[ { "panel_id": 93, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Stats OPS", "group": "Statistics", "used_for": "Used for monitoring the rate of statistics update operations in TiDB, broken down by update type. Essential for diagnosing statistics collection performance issues and understanding when auto-analyze jobs are running, which can impact query optimization and cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10029, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flush log duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log flush operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery mechanism when redo logs are being persisted to storage.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 14507, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Flush Size ", "group": "Backup Log", "used_for": "Used for monitoring the average size of backup log files flushed to storage in TiKV. Helps identify backup efficiency issues and diagnose abnormal flush patterns that could indicate storage bottlenecks or configuration problems in the log backup process.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]) / on(instance) increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 82, "dashboard": "TiDB-Cluster-Clinic", "title": "Keep Alive OPM", "group": "Server", "used_for": "Used for monitoring the rate of keep-alive operations per minute sent by TiDB instances to maintain connections with clients or other components. Helps diagnose connection management issues, detect abnormal connection churn, or identify instances experiencing connectivity problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 62, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CDC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiCDC (Change Data Capture) worker threads and TSO threads in TiKV instances. Essential for diagnosing CDC replication performance issues and identifying whether CDC workloads are causing CPU bottlenecks on specific TiKV nodes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10338, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush/L0 flow", "group": "Flow Control", "used_for": "**Used For:**\n\nUsed for monitoring the flow control metrics related to L0 SST files and memtable flush operations in TiKV's RocksDB engine. Essential for diagnosing write stalls and understanding when flow control throttling is triggered due to excessive L0 files or pending flush operations, which can impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "TiDB-Cluster-Clinic", "title": "Panic And Critial Error", "group": "Server", "used_for": "Used for monitoring panic events and critical errors occurring in TiDB server instances. Essential for detecting severe system failures and stability issues that require immediate attention, as these errors typically indicate serious bugs or system crashes that can impact cluster availability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 11, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted", "group": "", "used_for": "Used for monitoring the network transmission rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual outbound traffic patterns, or potential network bottlenecks affecting data replication and client communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 12895, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration Breakdown (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency breakdown of Raft Engine write operations across preprocessing, leader writing, and apply stages in TiKV. Essential for diagnosing write performance bottlenecks and identifying which specific phase of the Raft write pipeline is causing slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 634, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Input Rows", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the rate of rows being processed by TiCDC's transaction sink workers across different changefeeds. Essential for tracking data replication throughput and identifying bottlenecks in the sink pipeline when rows are not being consumed as expected.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10023, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo fsync duration", "group": "Redo", "used_for": "Used for monitoring the fsync operation latency in TiCDC's redo log system. Critical for diagnosing redo log write performance issues and identifying disk I/O bottlenecks that could impact data replication reliability and changefeed stability.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 60, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CDC endpoint CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of CDC (Change Data Capture) endpoint threads in TiKV instances. Essential for diagnosing CDC performance bottlenecks and identifying instances experiencing high CPU load during data replication to downstream systems.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdc_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 230, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load Latency 95 (ms)", "group": "Statistics", "used_for": "Used for monitoring the 95th percentile latency of statistics synchronization operations in TiDB, including both overall sync load time and stats reading time. Critical for diagnosing performance issues related to table statistics collection and synchronization, which can impact query optimization and execution plan generation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12898, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Written", "group": "Raft Engine", "used_for": "Used for monitoring the average, P99, and P999 bytes written per operation to TiKV's Raft Engine. Essential for diagnosing write amplification issues and identifying abnormally large Raft log entries that could impact storage efficiency and replication performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Entry Count", "group": "Raft Engine", "used_for": "Used for monitoring the average number of log entries stored in TiKV's Raft Engine, categorized by entry type. Helps diagnose Raft log accumulation issues, identify potential log compaction problems, and understand the distribution of different entry types that could impact storage and recovery performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(raft_engine_log_entry_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 51, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write regions across TiKV stores from a peer perspective. Helps identify uneven hot region distribution and potential write hotspot imbalances that could lead to performance bottlenecks on specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_peer\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received", "group": "", "used_for": "Used for monitoring the network ingress traffic rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual traffic patterns, or potential network bottlenecks affecting node communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 50, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's leader distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write regions across store leaders in the cluster. Helps identify load imbalance issues where certain stores are handling disproportionate write traffic, which can lead to performance bottlenecks and uneven resource utilization.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 229, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load QPS", "group": "Statistics", "used_for": "Used for monitoring the throughput of synchronous statistics loading operations in TiDB, including both successful loads and timeouts. Essential for diagnosing statistics collection performance issues and identifying when statistics loading is experiencing delays or failures that could impact query optimization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 81, "dashboard": "Dedicated Dataplane Node Info", "title": "Component Node Number Trends", "group": "", "used_for": "Used for tracking the number of active nodes for each TiDB component (TiDB, TiKV, TiFlash) over time in a dedicated dataplane deployment. Helps monitor cluster scaling events, detect unexpected node failures, and verify that the expected number of component instances are running.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "3", "namespace" ], "promql": [ "count(\n label_replace(\n kube_pod_info{namespace=\"$namespace\", pod=~\"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"},\n \"component\", \n \"$1$3\", \n \"pod\", \n \"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"\n )\n) by (component)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 61, "dashboard": "TiDB-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiDB instances. Essential for detecting goroutine leaks, resource exhaustion issues, and identifying abnormal concurrency patterns that could indicate connection pooling problems or stuck operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_sched_goroutines_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\"tidb.*\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12897, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Other Durations (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency of miscellaneous Raft Engine operations in TiKV, including reading entries, reading messages, and purging old data. Essential for diagnosing performance bottlenecks in Raft log management operations that may impact consensus protocol efficiency and overall cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(raft_engine_read_entry_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_read_message_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_purge_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 165, "dashboard": "TiDB-Cluster-Clinic", "title": "Prepare Statement Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of prepared statements currently cached in TiDB instances. Helps identify potential memory pressure from excessive prepared statement accumulation and diagnose connection pooling or statement lifecycle management issues in applications.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14362, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Flushed Size (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total amount of backup log data flushed to storage in the last 30 minutes across TiKV instances. Helps track backup progress, identify backup throughput issues, and diagnose problems with log backup operations when data volume appears abnormally low or high.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10027, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo write log duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log write operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery feature when redo logs are being written to external storage.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 691, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV backup worker threads responsible for handling backup operations. Essential for diagnosing backup performance issues, identifying CPU bottlenecks during backup tasks, and ensuring backup operations don't consume excessive resources that could impact normal cluster operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "BackGround Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV background worker threads (like compaction, cleanup, and maintenance tasks). Essential for identifying when background operations consume excessive CPU resources, potentially impacting foreground query performance or indicating storage engine bottlenecks.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10031, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write rows/s ", "group": "Redo", "used_for": "Used for monitoring the rate of rows written to TiCDC's redo log per second, broken down by instance and changefeed. Essential for tracking redo log write throughput and identifying performance bottlenecks or anomalies in TiCDC's disaster recovery mechanism during data replication.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[2m])) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12933, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL Duration Breakdown (999%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency breakdown of Raft Engine's Write-Ahead Log (WAL) operations, including leader write, log sync, and log allocation phases. Critical for diagnosing Raft log write performance bottlenecks and identifying which specific WAL operation stage is causing tail latency issues in TiKV's consensus layer.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_sync_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_allocate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_rotate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 233, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Healthy Distribution", "group": "Statistics", "used_for": "Used for monitoring the health status distribution of table statistics in TiDB. Essential for identifying stale or outdated statistics that can lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_statistics_stats_healthy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 635, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backend Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of DML batch commits to the downstream database in TiCDC's transaction sink. Critical for diagnosing slow replication issues and identifying backend database performance bottlenecks that affect change data capture throughput.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2531, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's garbage collection (GC) worker threads per instance. Essential for diagnosing GC performance issues, identifying instances with excessive GC overhead, and detecting potential bottlenecks in MVCC version cleanup operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14363, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flushed Files (Last 30m) Per Host", "group": "Backup Log", "used_for": "Used for monitoring the number of backup log files flushed to storage by each TiKV instance in the last 30 minutes. Helps track backup log activity patterns and identify instances with abnormal flush behavior or potential backup performance issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 628, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Outgoing Bytes", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the outgoing data throughput rate from TiCDC to Kafka message queues, measured in bytes per second. Essential for tracking replication bandwidth usage, identifying network bottlenecks, and ensuring data is being successfully transmitted to downstream Kafka clusters.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1463, "dashboard": "PD-Cluster-Clinic", "title": "Store witness count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of witness replicas stored on each TiKV store in the cluster. Essential for verifying witness replica distribution and diagnosing imbalanced placement of witness regions, which are lightweight replicas used for quorum voting without storing actual data.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"witness_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 14508, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Times (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the number of backup log flush operations completed in the last 30 minutes on TiKV instances. Helps track backup log activity frequency and identify potential issues with log backup file persistence or abnormal flush patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(sum(increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4559, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the p95 and average latency of encryption metadata read/write operations in TiKV. Critical for diagnosing performance overhead introduced by encryption-at-rest, helping identify if encryption metadata operations are causing slowdowns in encrypted TiKV clusters.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 653, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC workers sending messages to message queue (MQ) sinks like Kafka. Critical for diagnosing tail latency issues in data replication to downstream MQ systems and identifying potential bottlenecks in the CDC pipeline's message delivery performance.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 78, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's coprocessor threads across different priority levels (normal, high, low). Essential for diagnosing performance bottlenecks when TiDB pushes down computation to TiKV, helping identify whether coprocessor requests are consuming excessive CPU resources or experiencing priority-based contention.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 692, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU", "group": "Thread CPU", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of SST (Sorted String Table) import threads in TiKV instances. Essential for diagnosing performance issues during data import operations, snapshot ingestion, or when TiKV is receiving large amounts of data through bulk load processes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9947, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller actions", "group": "Flow Control", "used_for": "Used for monitoring the rate of flow control actions taken by TiKV's scheduler to prevent write overload. Essential for diagnosing write throttling issues and understanding when TiKV is actively limiting write operations to protect system stability under heavy I/O pressure.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_throttle_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cf, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Inaccuracy Rate", "group": "Statistics", "used_for": "**Used For:**\n\nUsed for monitoring the inaccuracy rate of table statistics at different percentiles (p50, p90, p99) in TiDB. Critical for identifying when statistics are outdated or inaccurate, which can lead to suboptimal query execution plans and performance degradation requiring manual ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9553, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler discard ratio", "group": "Flow Control", "used_for": "Used for monitoring the ratio of discarded scheduler commands in TiKV instances due to flow control mechanisms. Helps identify when TiKV is rejecting requests because of I/O pressure or resource constraints, indicating potential performance bottlenecks or overload conditions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_discard_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets on network interfaces during reception on dedicated dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that could impact cluster communication and data transfer reliability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14361, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Endpoint Status", "group": "Backup Log", "used_for": "Used for monitoring the operational status of TiKV's log backup endpoint, indicating whether log backup is enabled on each instance. Essential for verifying backup configuration and quickly identifying instances where log backup may be disabled or misconfigured.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_enabled{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 166, "dashboard": "TiDB-Cluster-Clinic", "title": "Time Jump Back OPS", "group": "Server", "used_for": "Used for detecting system clock backward jumps on TiDB instances, measured as occurrences per second. Critical for identifying time synchronization issues that can cause transaction timestamp conflicts, data consistency problems, and TSO (Timestamp Oracle) anomalies in distributed transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 11906, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler throttled CF", "group": "Flow Control", "used_for": "**Used For:**\n\nUsed for monitoring which column families (CFs) in TiKV's scheduler are being throttled due to flow control mechanisms. Helps identify write pressure and resource contention issues when the scheduler actively limits operations to specific column families to prevent overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_scheduler_throttle_cf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12899, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "File Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of log files, swap files, and recycled files in TiKV's Raft Engine storage layer. Essential for tracking file management health, identifying potential file accumulation issues, and diagnosing problems with log compaction or file recycling that could impact storage efficiency and performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(raft_engine_log_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "avg(raft_engine_swap_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "avg(raft_engine_recycled_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate bytes", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring write throughput (bytes per second) across TiKV stores to identify hot write regions and unbalanced write loads. Essential for diagnosing write hotspots, evaluating load distribution, and determining if specific stores are experiencing disproportionate write pressure that may require rebalancing.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 9552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler flow", "group": "Flow Control", "used_for": "Used for monitoring write flow and throttling activity in TiKV's scheduler across instances. Helps identify when flow control is actively throttling writes due to pressure, indicating potential write bottlenecks or resource constraints that may impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_write_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_throttle_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10025, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flushall duration", "group": "Redo", "used_for": "Used for monitoring the duration of redo log flush-all operations in TiCDC, which writes accumulated redo logs to persistent storage. Critical for diagnosing redo log write performance issues and identifying bottlenecks in TiCDC's disaster recovery mechanism that could impact changefeed replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 11512, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Throttle duration", "group": "Flow Control", "used_for": "Used for monitoring the cumulative time TiKV scheduler operations spend in throttled state due to flow control mechanisms. Essential for diagnosing performance degradation caused by resource pressure, helping identify when write operations are being intentionally delayed to prevent system overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_scheduler_throttle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 619, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the utilization ratio of TiCDC transaction sink workers, indicating how busy the workers are processing and flushing transactions to downstream. High ratios suggest workers are saturated and may become a bottleneck for replication throughput, while consistently low ratios indicate underutilization or light workload.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) /sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) *100" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 71, "dashboard": "TiDB-Cluster-Clinic", "title": "Pseudo Estimation OPS", "group": "Statistics", "used_for": "Used for monitoring the rate of queries using pseudo statistics (statistics that haven't been collected or are outdated) in TiDB. High values indicate tables lacking proper statistics, which can lead to suboptimal query plans and performance degradation requiring ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 58, "dashboard": "PD-Cluster-Clinic", "title": "Store leader count", "group": "Statistics - balance", "used_for": "Used for monitoring the distribution of Raft leaders across TiKV stores in the cluster. Essential for identifying leader imbalance issues that can cause uneven load distribution and performance hotspots, helping diagnose whether PD's leader scheduling is working effectively.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 14, "dashboard": "Dedicated Dataplane Node Info", "title": "Node labels", "group": "", "used_for": "Used for displaying Kubernetes node labels and metadata for dedicated dataplane nodes. Helps operators verify node configurations, identify node roles, and troubleshoot node-specific issues by correlating system metrics with Kubernetes label assignments.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "2", "instance", "k8s_cluster", "namespace", "pod" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node, k8s_cluster_info) group_right sum(kube_node_labels) by (node, k8s_cluster_info, label_zone, label_dedicated, label_failure_domain_beta_kubernetes_io_zone, label_beta_kubernetes_io_instance_type)", "(\n label_replace(\n node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"},\n \"node\", \"$0\", \"instance\", \".*\"\n )\n * on(node, k8s_cluster_info) group_right\n sum(kube_node_labels)\n by (node, k8s_cluster_info,\n label_zone, label_dedicated,\n label_failure_domain_beta_kubernetes_io_zone,\n label_beta_kubernetes_io_instance_type)\n)\n* on(node) group_left(pod, component)\n label_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\"$pod\"},\n \"component\", \"$2\", \"pod\",\n \"(^|.*-)(tidb|tikv|tiflash|pd|tso|ticdc)(-[0-9a-z]+)*$\"\n )" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 49, "dashboard": "TiDB-Cluster-Clinic", "title": "Events OPM", "group": "Server", "used_for": "Used for monitoring the rate of server events occurring in TiDB instances, measured as operations per minute over a 10-minute window. Helps identify abnormal event patterns, server state changes, or potential issues by tracking the frequency of internal server events like connection changes, schema updates, or other significant operational activities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12896, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow", "group": "Raft Engine", "used_for": "Used for monitoring the data flow rates in TiKV's Raft Engine, including write throughput and background rewrite operations by type. Essential for understanding Raft log write performance and identifying potential I/O bottlenecks or excessive rewrite activity that could impact cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_background_rewrite_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 160, "dashboard": "Prometheus-TiCDC-Clinic", "title": "GC STW Duration (last 256 GC cycles)", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring Go garbage collection stop-the-world (STW) pause durations in TiCDC instances over the last 256 GC cycles. Critical for diagnosing replication latency issues caused by excessive GC pauses that can block TiCDC's change data capture processing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1465, "dashboard": "PD-Cluster-Clinic", "title": "Total written bytes on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total write throughput (in bytes) on hot leader Regions across stores in the cluster. Essential for identifying write hotspots, detecting unbalanced write loads, and diagnosing performance bottlenecks caused by concentrated write traffic on specific leader Regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_bytes_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 14907, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Status", "group": "Backup Log", "used_for": "Used for monitoring the status of TiKV's log backup tasks across instances. Essential for quickly identifying backup task failures or abnormal states that could impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "min(tikv_log_backup_task_status{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "PD-Cluster-Clinic", "title": "Total written keys on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of keys written per second on hot leader Regions across stores. Helps identify write hotspots and unbalanced write loads, which can indicate the need for Region splitting or load rebalancing to prevent performance bottlenecks.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_keys_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 630, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Latency", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the latency of Kafka producer requests in TiCDC's message queue sink. Critical for diagnosing replication delays and identifying performance bottlenecks when TiCDC writes change data to Kafka topics.", "component": "TICDC", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 9962, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TSO Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TSO (Timestamp Oracle) worker threads in TiKV instances. Essential for diagnosing timestamp allocation performance issues and identifying CPU bottlenecks in the TSO service that could impact transaction processing across the cluster.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso_worker\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6986, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker compact duration", "group": "TTL", "used_for": "Used for monitoring the latency of TTL (Time To Live) checker compaction operations in TiKV at p95 and p99 percentiles. Essential for diagnosing performance issues with expired data cleanup and identifying when TTL compaction operations are taking longer than expected, which could impact storage efficiency and query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_ttl_checker_compact_duration_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_ttl_checker_compact_duration_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 235, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache OPS", "group": "Statistics", "used_for": "Used for monitoring the operations per second on TiDB's statistics cache, broken down by operation type (hits, misses, evictions). Essential for diagnosing statistics-related performance issues and evaluating the effectiveness of the stats cache in reducing metadata access overhead.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_stats_cache_op{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the number of live and obsolete blob files in TiKV's Titan storage engine. Essential for tracking Titan's blob file accumulation and garbage collection effectiveness, helping diagnose storage space issues and GC performance problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_num_obsolete_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_in_allowance_exceeded", "group": "", "used_for": "Used for monitoring network bandwidth ingress throttling events on dedicated dataplane nodes. Helps identify when incoming network traffic exceeds allocated bandwidth allowances, which can cause packet drops and network performance degradation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_in_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 709, "dashboard": "Prometheus-TiCDC-Clinic", "title": "encoder group input channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the input channel buffer size of TiCDC's encoder group when replicating data to message queue (MQ) sinks. Helps identify backpressure issues where events are queuing up before encoding, indicating potential bottlenecks in the encoding pipeline or downstream MQ write performance problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 168, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the total number of scheduler operations by stage and tracking \"too busy\" rejections for a specific command type in TiKV. Essential for diagnosing scheduler bottlenecks, understanding which processing stages are consuming resources, and identifying when TiKV is rejecting requests due to overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "TiDB-Cluster-Clinic", "title": "New ETCD Session Duration 95", "group": "Owner", "used_for": "Used for monitoring the 95th percentile latency of establishing new ETCD sessions in TiDB's owner election mechanism. Critical for diagnosing coordination issues, slow ETCD connections, or problems with DDL owner election that could impact schema changes and cluster management operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 56, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_out_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring network bandwidth throttling events when outbound traffic exceeds allocated limits on dedicated dataplane nodes. Critical for diagnosing network performance degradation caused by bandwidth quota enforcement and identifying nodes that may need increased bandwidth allocation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_out_allowance_exceeded{cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 191, "dashboard": "TiDB-Cluster-Clinic", "title": "Skip Binlog Count", "group": "Server", "used_for": "Used for monitoring the count of binlog write operations that were skipped due to critical errors in TiDB instances. Essential for diagnosing data replication issues and identifying when binlog generation failures may cause downstream synchronization problems with TiDB Binlog or CDC components.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "Dedicated Dataplane Node Info", "title": "conntrack_allowance_exceeded", "group": "", "used_for": "Used for monitoring connection tracking table overflow events on dedicated dataplane nodes. Helps diagnose network connection limits being exceeded, which can cause dropped connections and connectivity issues when the system attempts to track more connections than allowed.", "component": "", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_conntrack_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15361, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advancer Owner", "group": "Backup Log", "used_for": "Used for identifying which TiKV instance is currently serving as the log backup advancer owner. Essential for troubleshooting log backup coordination issues and verifying proper leader election in the backup subsystem.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "tidb_log_backup_advancer_owner > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 143, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate keys", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the write throughput in keys per second for each TiKV store, helping identify hot write stores and unbalanced write distribution. Essential for diagnosing write hotspots, evaluating load balancing effectiveness, and determining if specific stores are experiencing disproportionate write pressure that could impact cluster performance.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 236, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Replayer Task OPM", "group": "Statistics", "used_for": "Used for monitoring Plan Replayer task execution rates in TiDB, tracking both dump and capture operations per minute along with registered tasks. Essential for diagnosing issues with SQL plan reproduction and troubleshooting query optimization problems, as Plan Replayer helps capture execution context for problematic queries.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)", "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"capture\"}[2m])) by (result)", "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 629, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Rate", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of requests sent to Kafka brokers by TiCDC's message queue sink. Essential for tracking replication throughput to Kafka and diagnosing bottlenecks or connectivity issues in the data pipeline to downstream Kafka clusters.", "component": "TICDC", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572363, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time transaction commands spend being throttled due to flow control mechanisms in TiKV. Essential for diagnosing performance degradation caused by resource pressure, helping identify when transactions are being delayed by backpressure from storage layer congestion or quota limits.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_txn_command_throttle_time_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14135, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan Generate Event Throughput", "group": "Backup Log", "used_for": "Used for monitoring the throughput of initial scan operations during TiKV log backup, measured in bytes per second. Helps diagnose backup performance issues and track the rate at which historical data is being scanned and processed during the initial backup phase.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(tikv_log_backup_incremental_scan_bytes_sum{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4655, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the blob cache memory usage in TiKV's Titan storage engine across different column families. Essential for diagnosing memory consumption issues and optimizing Titan's blob cache configuration when handling large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "topk(20, avg(tikv_engine_blob_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by(cf, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10035, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Redo", "used_for": "Used for monitoring the busy ratio of TiCDC redo log workers, indicating how much time workers spend actively processing versus idle. Essential for identifying redo log processing bottlenecks and determining if workers are saturated, which could lead to replication lag or require scaling adjustments.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])/10) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 9946, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "Flow Control", "used_for": "Used for monitoring the volume of data waiting to be compacted in TiKV's RocksDB storage engine, broken down by column family. Critical for diagnosing write stalls and flow control issues, as excessive pending compaction bytes indicate the storage engine cannot keep up with write pressure, potentially triggering write throttling to prevent system overload.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"}) by (cf)", "max(tikv_scheduler_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimated Live Objects", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring the number of live objects in TiCDC's Go runtime heap memory. Helps diagnose memory leaks and excessive object allocation issues that could lead to increased garbage collection pressure and performance degradation.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 47, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets during network transmission on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that prevent packets from being sent successfully.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 4020, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache hit", "group": "Titan - $titan_db", "used_for": "Used for monitoring the cache hit rate of Titan's blob cache in TiKV. Helps diagnose blob storage performance issues by showing how effectively the cache is serving blob reads, where low hit rates indicate potential memory pressure or inefficient caching that could impact large value query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) / (sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) + sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_miss\"}[2m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6987, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker actions", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) checker actions in TiKV, broken down by action type. Helps diagnose TTL-related I/O activity and identify issues with expired data cleanup operations in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(rate(tikv_ttl_checker_actions{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 99, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Table Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the connection tracking table utilization ratio on dataplane nodes. Critical for identifying when the conntrack table approaches capacity limits, which can cause connection drops and network issues in high-traffic Kubernetes environments.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "node_nf_conntrack_entries{cluster=\"$k8s_cluster\",instance=~\"$instance\"} / node_nf_conntrack_entries_limit{cluster=\"$k8s_cluster\",instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 58, "dashboard": "Dedicated Dataplane Node Info", "title": "PPS_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring packet-per-second (PPS) rate limiting violations on network interfaces. Helps identify when network traffic exceeds configured PPS allowances, which can indicate traffic bursts, potential DDoS attacks, or insufficient network capacity provisioning.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_pps_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 55, "dashboard": "Dedicated Dataplane Node Info", "title": "linklocal_allowance_exceeded", "group": "", "used_for": "Used for monitoring the rate of link-local address allowance exceeded events on network interfaces in the dataplane node. Helps identify network configuration issues or excessive link-local traffic that may indicate connectivity problems or misconfigurations in the Kubernetes cluster networking layer.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_linklocal_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 12843, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle Event Rate", "group": "Backup Log", "used_for": "Used for monitoring the rate at which TiKV processes key-value batches during log backup operations. Helps identify backup throughput bottlenecks and diagnose performance issues when backup event processing slows down or stalls.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "rate(tikv_log_backup_handle_kv_batch_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9944, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller factors", "group": "Flow Control", "used_for": "Used for monitoring the key factors that trigger TiKV's write flow control mechanism, including L0 SST file count, memtable count, and average L0 files. Essential for diagnosing write throttling issues and understanding when TiKV applies backpressure to prevent storage engine overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_scheduler_l0{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_memtable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_l0_avg{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3523, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Live blob size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the total size of live (non-garbage) blob data stored in TiKV's Titan storage engine. Essential for tracking Titan storage utilization and identifying when blob garbage collection is needed to reclaim space from deleted or updated large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_live_blob_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1445, "dashboard": "PD-Cluster-Clinic", "title": "Store write query", "group": "Statistics - hot write", "used_for": "Used for monitoring the write query rate at the store level and identifying hot write regions in the cluster. Essential for diagnosing write hotspot issues, unbalanced load distribution across stores, and planning capacity when certain stores experience disproportionately high write traffic.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 167, "dashboard": "TiDB-Cluster-Clinic", "title": "Handshake Error OPS", "group": "Server", "used_for": "Used for monitoring the rate of client connection handshake failures per TiDB instance. Essential for diagnosing authentication issues, SSL/TLS configuration problems, or client compatibility issues that prevent successful database connections.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 13262, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Backup Log", "used_for": "Used for monitoring CPU usage of TiKV's backup log threads, including backup stream and log backup scan workers. Essential for identifying resource bottlenecks during continuous backup operations and diagnosing performance issues with point-in-time recovery (PITR) log backup processes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3557, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the storage space occupied by Titan blob files in TiKV, showing both live (active) and obsolete (garbage) blob file sizes. Essential for tracking Titan storage efficiency and identifying when garbage collection is needed to reclaim space from deleted large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_live_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_obsolete_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "PD-Cluster-Clinic", "title": "Selector write events", "group": "Statistics - hot write", "used_for": "Used for monitoring the rate of hot write region detection events in PD's scheduler. Helps diagnose write hotspot issues by tracking how frequently PD identifies regions with high write activity, which is essential for understanding load balancing and potential write bottlenecks in the cluster.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10033, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write bytes/s ", "group": "Redo", "used_for": "Used for monitoring the redo log write throughput (bytes per second) for each TiCDC instance. Essential for tracking redo log I/O performance and identifying instances with high write loads or potential I/O bottlenecks that could impact changefeed replication stability.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 163, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Watcher OPS", "group": "Owner", "used_for": "Used for monitoring the rate of owner watcher operations in TiDB's distributed coordination system. Helps diagnose issues with DDL owner election, schema synchronization, and cluster coordination by tracking watcher operation throughput and success/failure rates across instances.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 146, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache write entry numbers", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of hot write regions cached in PD's hotspot detection system. Essential for diagnosing write hotspot issues and understanding how many regions are experiencing concentrated write traffic that may require load balancing.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 631, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Records Per Request", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of records batched per Kafka produce request in TiCDC's MQ sink. Helps evaluate batching efficiency and diagnose throughput issues—low values may indicate poor batching performance, while high values suggest optimal message aggregation.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 171, "dashboard": "TiDB-Cluster-Clinic", "title": "RCCheckTS WriteConflict Num", "group": "Server", "used_for": "Used for monitoring the number of write conflicts detected during read-committed isolation level timestamp checks in TiDB. Essential for diagnosing transaction conflicts and concurrency issues that may cause transaction retries or failures in read-committed workloads.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_rc_check_ts_conflict_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572365, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Non-txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time non-transactional commands (like raw KV operations) are throttled due to flow control mechanisms in TiKV. Essential for diagnosing performance degradation when non-txn operations are being rate-limited, helping identify resource contention or quota enforcement issues.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_non_txn_command_throttle_time_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 234, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache Cost", "group": "Statistics", "used_for": "**Used For:**\n\nUsed for monitoring TiDB's statistics cache memory usage, tracking both current tracked items and total capacity. Essential for diagnosing memory pressure issues related to table statistics caching and determining if cache size adjustments are needed to optimize query plan generation.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"track\"}", "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"capacity\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 111, "dashboard": "TiDB-Cluster-Clinic", "title": "Get Token Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of token acquisition in TiDB's connection rate limiting mechanism. Critical for diagnosing connection throttling issues and identifying when clients experience delays due to token bucket rate limiting during high concurrency scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572511, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftlog fetch Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's raftlog fetch worker threads, which handle fetching Raft logs from the log storage engine. Essential for diagnosing performance bottlenecks when TiKV instances experience high CPU consumption during Raft log replication and identifying whether log fetching operations are becoming a resource constraint.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftlog_fetch.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 157, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Usage", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory consumption, including resident memory, heap allocation, and GC thresholds. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC's memory configuration during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 627, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Inflight Requests", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of in-flight (pending) requests to Kafka brokers from TiCDC's MQ sink. Critical for identifying backpressure issues, network bottlenecks, or Kafka broker slowness that could cause replication lag or changefeed stalls.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 626, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Producer Compression Ratio", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the compression efficiency of data sent to Kafka by TiCDC, showing both average and 99th percentile compression ratios. Essential for evaluating network bandwidth utilization and identifying changefeeds with poor compression performance that may impact throughput and storage costs.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", "histogram_quantile(0.99, sum(rate(ticdc_kafka_sink_codec_compression_ratio_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 237, "dashboard": "TiDB-Cluster-Clinic", "title": "Historical Stats OPM", "group": "Statistics", "used_for": "Used for monitoring the operations per minute (OPM) of TiDB's historical statistics collection, tracking both generation and dump operations with their success/failure rates. Essential for diagnosing issues with statistics maintenance jobs and ensuring the query optimizer has up-to-date table statistics for optimal execution plans.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[2m])) by (result)", "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 69, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring transaction operations per second in TiDB, broken down by transaction type (commit/rollback) and mode (optimistic/pessimistic), including both general user transactions and internal system transactions. Essential for tracking transaction throughput, identifying transaction pattern changes, and diagnosing issues related to transaction processing capacity or abnormal commit/rollback ratios.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (type, txn_mode)", "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"internal\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 211, "dashboard": "TiDB-Cluster-Clinic", "title": "Client Data Traffic", "group": "Server", "used_for": "Used for monitoring the network traffic volume between TiDB servers and clients, broken down by traffic type (inbound/outbound). Essential for identifying network bottlenecks, unusual traffic patterns, and diagnosing connectivity issues or data transfer anomalies.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14911, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Files (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total number of log backup files flushed to external storage in the last 30 minutes across TiKV instances. Helps diagnose backup progress issues, identify stalled backup operations, and verify that log backup is actively writing data to the backup destination.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(sum(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6985, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL check progress", "group": "TTL", "used_for": "Used for monitoring the progress of Time-To-Live (TTL) checks across TiKV instances by showing the ratio of processed regions to total regions. Helps track TTL scan completion and identify instances that are lagging behind in expired data cleanup operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(tikv_ttl_checker_processed_regions{instance=~\"$instance\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 150, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot write", "used_for": "Used for monitoring the direction and frequency of hot write region leader transfers between TiKV stores. Helps diagnose whether PD's hotspot scheduling is actively balancing write-heavy regions and identify stores that are frequently transferring leaders in or out due to write hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 710, "dashboard": "Prometheus-TiCDC-Clinic", "title": "encoder group output channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the output channel buffer size of TiCDC's encoder group when replicating to message queue sinks. Helps identify backpressure issues and bottlenecks in the encoding pipeline when the channel fills up, indicating that downstream MQ consumption is slower than upstream encoding.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 161, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Allocator Throughput", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory allocation throughput, including allocation rate, deallocation rate, and object allocation frequency. Essential for diagnosing memory pressure issues, identifying memory leaks, and understanding garbage collection behavior in TiCDC runtime instances.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the P95 and P99 latency of TiKV scheduler commands (such as prewrite, commit, or read operations). Essential for diagnosing transaction processing delays and identifying performance bottlenecks in the storage layer's command execution pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7326, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker poll interval", "group": "TTL", "used_for": "Used for monitoring the polling interval of TiKV's TTL (Time-To-Live) checker, which determines how frequently expired data is scanned and cleaned up. Essential for diagnosing TTL cleanup performance issues and understanding the trade-off between cleanup frequency and system overhead.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "max(tikv_ttl_checker_poll_interval{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14774, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Abnormal Checkpoint TS Lag", "group": "Backup Log", "used_for": "Used for monitoring the time lag between the current time and the last backup checkpoint timestamp in TiDB log backup tasks. Critical for detecting backup delays or stalls that could impact point-in-time recovery capabilities and data protection SLAs.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "time() * 1000 - max(tidb_log_backup_last_checkpoint / 262144 > 0) by (task)", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 98, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Drops", "group": "", "used_for": "Used for monitoring the rate of dropped connection tracking entries in the Linux kernel's netfilter conntrack subsystem. Critical for diagnosing network connectivity issues caused by conntrack table exhaustion, which can lead to dropped connections and failed network requests when the system reaches its connection tracking limits.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13100, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Of Events", "group": "Backup Log", "used_for": "Used for monitoring the heap memory consumption of TiKV's backup log events. Helps identify memory pressure issues during log backup operations and diagnose potential out-of-memory problems in the backup subsystem.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_heap_memory{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 711, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of batch processing in TiCDC's MQ (Message Queue) sink workers. Critical for identifying tail latency issues when TiCDC replicates data to downstream message queue systems like Kafka, helping diagnose slow batch operations that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 50, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID QPS", "group": "Meta", "used_for": "Used for monitoring the rate of AutoID allocation operations per second in TiDB. Helps identify issues with auto-increment ID generation performance and diagnose bottlenecks when applications experience slow inserts due to ID allocation delays.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time transactions spend waiting to acquire memory locks (latches) in TiKV's scheduler before executing commands. Critical for diagnosing transaction contention issues and identifying hotspot keys that cause concurrent transactions to wait, which directly impacts write performance and transaction latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile transaction duration in TiDB, broken down by transaction type (optimistic/pessimistic) and showing both percentile and average latencies. Essential for identifying transaction performance degradation, comparing different transaction modes, and diagnosing issues when users report slow commit times or transaction timeouts.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, txn_mode))", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 148, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot write", "used_for": "Used for tracking PD's hotspot scheduling decisions for write-heavy regions, showing the direction and frequency of peer/leader movements across stores. Essential for diagnosing whether PD is actively balancing write hotspots and identifying stores that are frequently sources or destinations of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10001, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of DDL schema changes published to Pulsar topics by TiCDC. Essential for tracking schema evolution activity and diagnosing issues when DDL operations aren't being properly replicated to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1441, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot write", "used_for": "Used for monitoring the number of pending items in PD's hot write region detection queue. Helps diagnose when hot region identification is lagging behind actual workload, which can delay load balancing and affect write performance during traffic spikes.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 14630, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observed Region Count", "group": "Backup Log", "used_for": "Used for monitoring the number of regions being observed by TiKV's backup log feature across instances. Helps diagnose backup coverage issues and verify that all regions are properly tracked during log backup operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_observed_region{instance=~\"$instance\"}", "sum(tikv_log_backup_observed_region{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Statement Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of statement counts per transaction in TiDB. Helps identify transactions with excessive statements that may cause performance issues, long-running transactions, or inefficient batch operations requiring optimization.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3414, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob key size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob keys in TiKV's Titan storage engine, showing average, P95, and P99 key sizes. Helps diagnose storage efficiency issues and identify whether large keys are impacting Titan's performance or causing excessive memory usage during blob file operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_average\"})", "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile95\"})", "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile99\"})", "max(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13101, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errors", "group": "Backup Log", "used_for": "Used for monitoring error occurrences in TiKV's log backup process. Essential for identifying backup failures, detecting reliability issues in the backup pipeline, and troubleshooting when continuous data protection is compromised.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__interval", "instance" ], "promql": [ "increase(tikv_log_backup_errors{instance=~\"$instance\"}[$__interval])", "tikv_log_backup_errors{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4023, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Iter touched blob file count", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the number of blob files accessed during Titan iterator operations in TiKV's Titan storage engine. Helps diagnose scan performance issues and assess the efficiency of blob file organization, as higher counts may indicate fragmentation or suboptimal blob file layout affecting range query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_average\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile95\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile99\"})", "max(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1069, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of requests sent from TiKV to PD (Placement Driver). Helps identify communication patterns between TiKV and PD, and diagnose issues when TiKV cannot properly interact with PD for metadata operations, region scheduling, or TSO allocation.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4025, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file discardable ratio distribution", "group": "Titan - $titan_db", "used_for": "Used for monitoring the distribution of discardable data ratios across Titan blob files in TiKV's storage engine. Helps identify garbage collection efficiency and determine when blob files contain enough obsolete data to warrant compaction, which is critical for managing disk space and I/O performance in Titan-enabled clusters.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_blob_file_discardable_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by (ratio)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 712, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Size Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile batch size of messages sent by TiCDC's MQ sink workers to message queues like Kafka. Critical for diagnosing message batching efficiency and identifying whether the sink is properly aggregating changes before sending, which directly impacts throughput and downstream processing performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 36, "dashboard": "TiDB-Cluster-Clinic", "title": "Session Retry Error OPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate of transaction retry errors in TiDB sessions, categorized by error type and SQL type. Essential for diagnosing transaction conflicts, lock contention issues, and identifying which types of queries are experiencing retry failures that could impact application performance.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, sql_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 101, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Early Drops", "group": "", "used_for": "Used for monitoring the rate of connection tracking table entries dropped due to table overflow on dataplane nodes. Critical for diagnosing network connection limits and potential packet loss when the conntrack table reaches capacity under high connection loads.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_early_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 10003, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring DDL schema publication failures when TiCDC replicates data to Pulsar. Critical for diagnosing issues where DDL changes fail to be published to Pulsar topics, which can cause replication inconsistencies or changefeed failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 731, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Message Count Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the distribution of message counts per batch in TiCDC's Kafka producer when using the Kafka-Go client. Helps diagnose batching efficiency and identify whether messages are being properly aggregated before sending to Kafka, which impacts throughput and network utilization.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10002, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication of DDL schema changes to Pulsar topics in TiCDC. Essential for verifying that database schema modifications are correctly propagated to downstream Pulsar consumers and identifying schema synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 100, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Insert Failures", "group": "", "used_for": "Used for monitoring the rate of connection tracking table insertion failures on dataplane nodes. Critical for diagnosing network connection issues caused by conntrack table exhaustion, which can lead to dropped connections and increased latency when the system cannot track new network flows.", "component": "", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_insert_failed{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 52, "dashboard": "TiDB-Cluster-Clinic", "title": "Meta Operations Duration 99", "group": "Meta", "used_for": "Used for monitoring the 99th percentile latency of TiDB metadata operations (such as schema changes, table creation, and DDL operations). Critical for diagnosing slow DDL execution and identifying metadata-related performance bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 195, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiKV at p95 and p99 percentiles. Essential for identifying commands that scan excessive keys, which can indicate inefficient queries, large range scans, or potential hotspot issues affecting I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 733, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Messages Duration Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the duration percentiles of Kafka message batching operations in TiCDC's Kafka-Go producer implementation. Critical for diagnosing message batching latency issues and identifying performance bottlenecks in the CDC replication pipeline to Kafka sinks.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_duration{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 67, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction retry attempts in TiDB. Helps diagnose transaction conflicts, write contention issues, and identify workloads experiencing frequent optimistic lock failures that may benefit from pessimistic locking or application-level optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3446, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob value size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob values stored in TiKV's Titan storage engine, showing average, P95, and P99 metrics. Essential for understanding large value storage patterns and optimizing Titan's blob file management when dealing with values exceeding the separation threshold.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_average\"})", "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile95\"})", "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile99\"})", "max(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 373, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiKV, showing p95, p99, and average metrics. Essential for identifying write-heavy operations and diagnosing performance issues caused by commands that modify an excessive number of keys in a single transaction.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 51, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID Duration", "group": "Meta", "used_for": "Used for monitoring the latency of TiDB's auto-increment ID allocation operations at p80 and p99 percentiles. Critical for diagnosing performance issues when applications experience delays in obtaining auto-increment IDs, particularly during high-concurrency insert workloads or when the auto-ID cache needs replenishment from PD.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14910, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checkpoint TS of Tasks", "group": "Backup Log", "used_for": "**Used For:**\n\nUsed for monitoring the checkpoint timestamp of log backup tasks in TiKV, showing how current each backup task is relative to real-time. Essential for identifying backup lag issues and ensuring point-in-time recovery capabilities are maintained within acceptable time windows.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "max(tidb_log_backup_last_checkpoint{instance=~\"$instance\"} / 262144 > 0) by (task)", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot read", "used_for": "Used for monitoring the distribution of hot read regions across TiKV stores, showing both peer and leader roles. Helps identify load imbalance issues where hot regions are concentrated on specific stores, enabling targeted rebalancing decisions to optimize read performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_peer\"}", "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10004, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the rate of DML schema changes published to Pulsar topics by TiCDC. Helps identify schema evolution activity and troubleshoot issues with schema synchronization to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3746, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob retrieval operations in TiKV's Titan storage engine. Essential for tracking Titan's blob file access patterns and diagnosing performance issues related to large value reads when Titan is enabled.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_get\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 735, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Bytes Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the maximum batch size (in bytes) of messages sent to Kafka by TiCDC's kafka-go producer library. Helps diagnose message batching efficiency and identify potential throughput bottlenecks when replicating data to Kafka sinks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_size{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1396, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiKV and PD (Placement Driver). Helps diagnose cluster membership issues, region metadata synchronization problems, and potential communication failures between TiKV nodes and PD.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1070, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different types of requests from TiKV to PD (Placement Driver). Essential for diagnosing communication issues between TiKV and PD, identifying slow PD operations like TSO allocation or region heartbeats that could impact cluster coordination and transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "Used for monitoring the current memory utilization on dedicated dataplane nodes. Essential for identifying memory pressure, potential OOM risks, and capacity planning for node resources.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(\n (\n (\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n )\n / node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n ) * 100)\n" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 10005, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication rate of DML schema changes to Pulsar topics in TiCDC. Helps verify that schema changes are being correctly propagated to downstream Pulsar systems and identify potential schema synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 105, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate bytes", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in bytes per second across TiKV stores, comparing actual store read rates with expected hot region read rates. Essential for identifying read hotspots, diagnosing unbalanced read traffic distribution, and validating whether PD's hot region scheduling is effectively balancing read-intensive workloads across stores.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 4, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring the throughput of KV transaction operations in TiDB, broken down by operation type (commit, rollback, etc.) and instance. Essential for tracking transaction workload patterns and identifying instances with abnormal transaction rates or potential bottlenecks in the distributed transaction layer.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1215, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate and volume of heartbeat messages between TiKV instances and PD (Placement Driver). Critical for diagnosing cluster coordination issues, detecting communication failures between TiKV and PD, and identifying heartbeat backlogs that may indicate network problems or PD overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 560, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's scheduler, broken down by operation tags. Helps diagnose scan-heavy workloads and identify which types of scan operations (e.g., range scans, index lookups) are consuming resources, useful for optimizing query patterns and detecting inefficient table scans.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3655, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob retrieval operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately from the LSM tree, helping identify blob file access bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14908, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fatal Errors", "group": "Backup Log", "used_for": "Used for monitoring fatal errors in TiKV's backup log functionality. Critical for detecting severe backup failures that require immediate attention, helping diagnose issues that could lead to data loss or backup chain interruption.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__interval", "instance" ], "promql": [ "increase(tikv_log_backup_fatal_errors{instance=~\"$instance\"}[$__interval])", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 196, "dashboard": "TiDB-Cluster-Clinic", "title": "Commit Token Wait Duration", "group": "Transaction", "used_for": "Used for monitoring the wait time transactions spend acquiring commit tokens in TiDB's batch executor at p80, p95, and p99 percentiles. Critical for diagnosing transaction commit bottlenecks caused by token contention when concurrent commits exceed available token capacity, which can indicate the need to adjust `committer-concurrency` settings.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 107, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate keys", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in terms of keys per second for each TiKV store, helping identify hot read regions and unbalanced read loads across stores. Essential for diagnosing read hotspots and determining if read traffic is evenly distributed in the cluster.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 6, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "Used for monitoring memory consumption on dedicated dataplane nodes, breaking down total used memory, buffer cache, and page cache. Essential for identifying memory pressure, detecting potential OOM risks, and understanding how the system allocates memory between applications and kernel caches.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "node", "port" ], "promql": [ "max(\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n)\n", "max(node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$node:$port\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 675, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations specifically on the lock column family during scheduler command execution in TiKV. Essential for diagnosing transaction conflicts and lock contention issues by tracking how frequently the scheduler scans lock-related data, helping identify bottlenecks in transaction processing.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 739, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Retry Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of message retry attempts in TiCDC's Kafka sink when using the kafka-go client library. Helps identify message delivery issues, network instability, or Kafka broker problems that cause TiCDC to retry sending change events to Kafka topics.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_retry_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 85, "dashboard": "TiDB-Cluster-Clinic", "title": "Worker Action OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) worker operations per minute in TiDB, broken down by action type. Helps diagnose GC performance issues, identify bottlenecks in the cleanup process, and understand which GC operations (like resolve locks, delete ranges) are most active during troubleshooting.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14078, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to flush backup log files to external storage in TiKV. Critical for diagnosing backup performance issues and identifying bottlenecks in the log backup pipeline's file-saving stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_flush_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_files\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7985, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD reconnections", "group": "PD", "used_for": "Used for monitoring the rate of reconnection attempts between TiKV instances and PD (Placement Driver) servers. Critical for diagnosing network instability, PD availability issues, or connection problems that could impact cluster coordination and metadata operations.", "component": "TIKV", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_pd_reconnect_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 829, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's write column family during scheduler command execution. Helps diagnose write-related performance issues by revealing scan patterns and identifying whether excessive scanning is causing I/O bottlenecks or slow write operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 193, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution of KV transaction operations in TiDB at the 80th, 95th, and 99th percentiles. Essential for diagnosing transaction performance issues, identifying slow transaction types (commit, prewrite, etc.), and detecting latency degradation that impacts application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3338, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob iter operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of iterator operations (seek, prev, next) on Titan blob files in TiKV's storage engine. Essential for diagnosing Titan blob storage I/O patterns and identifying potential performance bottlenecks when scanning large values stored separately from the LSM tree.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_seek\"}[2m]))", "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_prev\"}[2m]))", "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_next\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 86, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99", "group": "GC", "used_for": "Used for monitoring the 99th percentile latency of TiDB's garbage collection (GC) operations. Essential for identifying GC performance degradation that could impact transaction processing and detecting when old MVCC versions are taking too long to clean up.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 737, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Error Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's Kafka-Go producer when replicating data to Kafka sinks. Critical for identifying replication failures, connection issues, or message delivery problems that could cause data loss or replication lag.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_err_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1446, "dashboard": "PD-Cluster-Clinic", "title": "Store read query", "group": "Statistics - hot read", "used_for": "Used for monitoring read query rates on TiKV stores to identify hot read regions and unbalanced read traffic distribution. Essential for diagnosing read hotspots, evaluating load balancing effectiveness, and determining if specific stores are handling disproportionate read workloads that could impact cluster performance.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10006, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring failed DML schema publications to Pulsar topics in TiCDC. Critical for identifying schema synchronization issues when replicating data changes to Pulsar, helping diagnose why certain table schemas fail to publish and may cause replication delays or data inconsistencies.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 14136, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scanning duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to complete initial log scanning during TiKV backup operations. Helps diagnose slow backup startup issues and identify performance bottlenecks in the initial data scanning phase of log backup.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_initial_scan_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Regions Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of region counts involved in transaction prewrite operations. Helps identify transactions that span many regions, which can indicate inefficient data access patterns or potential performance bottlenecks in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8376, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD forward status", "group": "PD", "used_for": "Used for monitoring whether TiKV's PD client requests are being forwarded to other PD nodes. Helps diagnose PD leader election issues, network connectivity problems between TiKV and PD, or misconfigurations where requests aren't reaching the correct PD leader.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_pd_request_forwarded{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Convert Raft Event duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent converting Raft events to stream events during TiKV log backup operations. Critical for diagnosing backup performance bottlenecks and identifying delays in the event conversion pipeline that could impact backup throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"to_stream_event\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "TiDB-Cluster-Clinic", "title": "GC Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection failures per minute in TiDB, categorized by failure type. Critical for identifying GC issues that can lead to storage bloat, performance degradation, and potential cluster instability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 740, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of claim check message sending operations in TiCDC's MQ sink. Critical for diagnosing tail latency issues when TiCDC uses claim check pattern to handle large messages in message queue sinks, helping identify performance bottlenecks in external storage interactions.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "rate(ticdc_sink_mq_claim_check_send_message_duration_sum[30s]) / rate(ticdc_sink_mq_claim_check_send_message_duration_count[30s])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3412, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob seek duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of seek operations in TiKV's Titan blob storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately in Titan's blob files, helping identify slow blob file access patterns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572710, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command read duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the P95 and P99 latency of read operations in TiKV's scheduler for specific command types. Essential for diagnosing slow read performance issues and identifying whether the scheduler's read processing is causing query latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "Dedicated Dataplane Node Info", "title": "System load", "group": "CPU", "used_for": "Used for monitoring the 1-minute, 5-minute, and 15-minute system load averages on dedicated dataplane nodes. Essential for identifying CPU resource saturation and determining if the node is overloaded or experiencing sustained high workload pressure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(node_load1{job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load5{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load15{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 87, "dashboard": "TiDB-Cluster-Clinic", "title": "Config", "group": "GC", "used_for": "Used for monitoring TiDB's garbage collection (GC) configuration parameters across the cluster. Helps verify GC settings are correctly applied and identify configuration inconsistencies that could affect data cleanup and storage reclamation.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10007, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client bytes published ", "group": "Pulsar Sink", "used_for": "Used for monitoring the total bytes published by TiCDC's Pulsar client to the Pulsar message queue, broken down by changefeed and instance. Essential for tracking data replication throughput to Pulsar and diagnosing network bandwidth issues or replication bottlenecks in the Pulsar sink pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 830, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's default column family, broken down by scan type tags. Essential for diagnosing scan performance issues, identifying inefficient range queries, and understanding read patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10008, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client connections opened", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of active Pulsar client connections opened by TiCDC changefeeds. Helps diagnose connection management issues, detect connection leaks, and verify proper connectivity between TiCDC and Pulsar message brokers during data replication.", "component": "TICDC", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "Dedicated Dataplane Node Info", "title": "Usage Per Core", "group": "CPU", "used_for": "Used for monitoring CPU utilization per individual core on dedicated dataplane nodes. Helps identify CPU core imbalances, hotspots, and whether workloads are properly distributed across all available cores.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "sum by (cpu) (irate(node_cpu_seconds_total{ job=\"node-exporter\", mode!=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m]))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 9963, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending TSO Requests", "group": "PD", "used_for": "Used for monitoring the number of TSO (Timestamp Oracle) requests currently pending in TiKV instances waiting for PD response. Critical for diagnosing PD service bottlenecks, network latency issues, or TSO allocation problems that can cause transaction delays across the cluster.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_pd_pending_tso_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4, "dashboard": "Dedicated Dataplane Node Info", "title": "CPU Utilization", "group": "CPU", "used_for": "Used for monitoring the overall CPU utilization percentage on dedicated dataplane nodes. Essential for identifying CPU bottlenecks, resource saturation, and determining if nodes need scaling or workload rebalancing.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m])) * 100)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 7236, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check memory locks duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time spent checking memory locks during transaction processing in TiKV's scheduler. Critical for diagnosing transaction conflicts and lock contention issues that can cause increased latency in write-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_check_mem_lock_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 149, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot read", "used_for": "Used for monitoring the direction and frequency of hot read region leader transfers across TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing read-heavy regions and identify stores that are frequently transferring leaders in or out to resolve read hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12840, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait for Lock Duration", "group": "Backup Log", "used_for": "Used for monitoring the time TiKV spends waiting to acquire router locks during backup log operations. Critical for diagnosing backup performance bottlenecks caused by lock contention in the log backup pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"get_router_lock\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob next duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob iterator next operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when scanning large values stored separately in Titan's blob files, helping identify slow blob file reads that impact range query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 741, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Count", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of claim check messages sent by TiCDC's MQ sink when handling large messages. Helps diagnose message overflow scenarios where TiCDC offloads oversized messages to external storage instead of sending them directly to the message queue.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10365, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store Slow Score", "group": "PD", "used_for": "Used for monitoring the slow score of each TiKV store, which indicates how slow a store is responding compared to others in the cluster. Essential for identifying problematic TiKV instances that may be experiencing disk I/O issues, hardware degradation, or becoming performance bottlenecks that could trigger PD's store rebalancing mechanisms.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_raftstore_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 147, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache read entry numbers", "group": "Statistics - hot read", "used_for": "Used for monitoring the total number of hot read regions cached in PD's hotspot detection system. Essential for understanding read hotspot distribution across stores and diagnosing unbalanced read traffic or hotspot scheduling issues.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10009, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client rpc count", "group": "Pulsar Sink", "used_for": "Used for monitoring the total number of RPC calls made by TiCDC's Pulsar client to the Pulsar cluster. Helps diagnose communication issues between TiCDC and Pulsar, identify connection problems, and track the volume of interactions when troubleshooting data replication delays or failures in the Pulsar sink.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 158, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Failure OPM", "group": "GC", "used_for": "Used for monitoring the per-minute failure rate of delete range operations during TiDB's garbage collection process, categorized by failure type. Critical for identifying issues with physical data deletion in TiKV that could lead to disk space not being reclaimed properly or GC process degradation.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 33, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative count of key-value pairs written per transaction in TiDB. Helps identify transaction write patterns, detect abnormal batch sizes, and diagnose issues related to large transactions that may impact cluster performance or cause write hotspots.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4093, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob prev duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob previous operations in TiKV's Titan storage engine, which handles large value storage. Critical for diagnosing performance issues when iterating backwards through blob files, showing average, P95, and P99 latencies to identify slow blob traversal operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 201, "dashboard": "TiDB-Cluster-Clinic", "title": "Statement Lock Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of lock keys acquired per statement in TiDB transactions. Helps diagnose lock contention issues and identify statements that acquire excessive locks, which can lead to transaction conflicts and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1440, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot read", "used_for": "Used for monitoring the number of pending items in PD's hot read region detection queue. Helps diagnose delays in hot region identification and potential bottlenecks in PD's hot region scheduling mechanism when the queue accumulates items.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 714, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write Bytes/s", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the write throughput (bytes per second) of TiCDC's cloud storage sink. Essential for tracking data replication performance to cloud storage and identifying bandwidth bottlenecks or write slowdowns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10366, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Inspected duration per server", "group": "PD", "used_for": "Used for monitoring the 99th percentile duration of Raft store inspection operations per TiKV instance. Critical for diagnosing performance issues in Raft consensus protocol health checks and identifying instances with slow inspection latency that may impact cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_inspect_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 45, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk I/O Util", "group": "Disk", "used_for": "Used for monitoring disk I/O utilization percentage on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting saturated storage devices, and diagnosing performance degradation caused by excessive I/O operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 90, "dashboard": "TiDB-Cluster-Clinic", "title": "Too Many Locks Error OPM", "group": "GC", "used_for": "Used for monitoring the rate of \"too many locks\" errors encountered during garbage collection operations in TiDB. Helps diagnose GC blocking issues caused by excessive uncommitted locks in regions, which can prevent old data cleanup and lead to storage bloat.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12841, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Save to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent saving backup log data to temporary files in TiKV. Critical for diagnosing performance bottlenecks in the log backup pipeline, particularly when backup operations are slow or experiencing delays in the initial file writing stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_to_temp_file\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 144, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot read", "used_for": "Used for tracking the direction and frequency of hot region movements (peer and leader transfers) across TiKV stores during read-heavy workloads. Essential for diagnosing whether PD's hotspot scheduler is effectively balancing read hotspots by moving regions away from overloaded stores to underutilized ones.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3645, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob keys flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob key operations in TiKV's Titan storage engine, tracking both keys written to and read from blob files. Essential for diagnosing Titan's key-value separation performance and identifying issues with blob file key management or unusual access patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3643, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob bytes flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of data flow in TiKV's Titan storage engine, tracking bytes written to and read from blob files. Essential for understanding Titan's I/O patterns and diagnosing performance issues related to large value storage when key-value separation is enabled.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 715, "dashboard": "Prometheus-TiCDC-Clinic", "title": "File Count", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the total number of files generated by TiCDC's cloud storage sink for each changefeed instance. Helps track file accumulation patterns and identify potential issues with file generation rate or storage cleanup processes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 209, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of key-value pairs written per transaction in TiDB. Helps identify transactions with abnormally high write volumes that may cause performance bottlenecks or exceed transaction size limits.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1466, "dashboard": "PD-Cluster-Clinic", "title": "Selector read events", "group": "Statistics - hot read", "used_for": "Used for monitoring the rate of hot read region detection events in PD's scheduler. Helps identify when PD is actively tracking read hotspots across stores, which is essential for diagnosing read traffic imbalances and evaluating the effectiveness of hot region scheduling policies.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10011, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer rpc latency", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile RPC latency of Pulsar client producers in TiCDC's Pulsar sink. Critical for identifying tail latency issues when TiCDC replicates change data to Pulsar, helping diagnose network delays or Pulsar broker performance problems that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent writing backup log data to temporary files in TiKV. Critical for diagnosing backup performance bottlenecks and identifying I/O issues during the log backup process.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=\"write_to_tempfile\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10010, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer latency ", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC's Pulsar client producer when writing change data to Pulsar message queues. Critical for diagnosing replication delays and identifying performance bottlenecks in the TiCDC-to-Pulsar data pipeline, especially when changefeeds experience slow message publishing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r", "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3834, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the volume of data being written by TiKV's scheduler component on each instance. Helps identify write-heavy instances, diagnose unbalanced write distribution across the cluster, and detect potential write bottlenecks or abnormal write traffic patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of write I/O throughput in bytes per second across TiKV instances, broken down by operation type. Essential for diagnosing write performance bottlenecks, identifying which storage operations consume the most I/O bandwidth, and detecting abnormal disk write patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[45s]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 167, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiKV, including both normal operations and \"too busy\" rejections. Essential for diagnosing scheduler bottlenecks, identifying when TiKV is overloaded and rejecting requests, and understanding the distribution of work across scheduler stages like snapshot, apply, and write.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15059, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Command Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of key-value batch sizes processed during TiKV log backup operations. Helps identify batch processing efficiency and diagnose potential performance issues when backup commands are handling unusually large or small batches of data.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_handle_kv_batch_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 194, "dashboard": "TiDB-Cluster-Clinic", "title": "Send HeartBeat Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of transaction heartbeat operations sent from TiDB to TiKV at p80, p95, and p99 percentiles. Critical for diagnosing issues with long-running transactions that require periodic heartbeats to prevent lock timeouts, especially when investigating transaction commit delays or lock wait problems.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 89, "dashboard": "TiDB-Cluster-Clinic", "title": "Action Result OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) action results per minute in TiDB, categorized by operation type. Helps diagnose GC efficiency issues, identify failed GC operations, and track the health of TiDB's distributed garbage collection process that cleans up obsolete MVCC versions.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3657, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file read duration", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring read latency of blob files in TiKV's Titan storage engine, showing average, P99, and P95 durations. Critical for diagnosing performance issues when reading large values stored separately from the LSM tree, helping identify I/O bottlenecks in Titan's blob file operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_average\"})", "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile99\"})", "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile95\"})", "max(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 34, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative volume of data written by TiDB transactions to TiKV. Essential for analyzing transaction write throughput patterns, identifying write-heavy workloads, and diagnosing performance issues related to large transaction sizes or sudden increases in write volume.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiKV. Helps identify workload distribution across different priority queues and diagnose issues where high-priority commands may be blocked by lower-priority operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 181, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Task Status", "group": "GC", "used_for": "Used for monitoring the status and results of delete range tasks in TiDB's garbage collection process. Helps diagnose issues with space reclamation after table/index drops, identifying failed or stuck delete operations that could lead to storage bloat.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 193, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands queued in each TiKV instance. Helps identify scheduler bottlenecks and command processing delays that could indicate storage layer congestion or resource saturation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13551, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "System Write Call Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration distribution of system write calls during TiKV log backup operations. Essential for diagnosing I/O performance bottlenecks and identifying when disk write latency impacts backup log processing efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=\"syscall_write\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3408, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file write duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring write latency to Titan blob files in TiKV's storage engine, showing average, P99, and P95 durations. Critical for diagnosing performance issues when large values are stored separately in Titan's blob storage, helping identify slow writes that could impact overall TiKV write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_average\"})", "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile99\"})", "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile95\"})", "max(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10012, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in TiCDC's Pulsar producer queue that are waiting to be sent to Pulsar. Critical for identifying message backlog issues, producer throughput bottlenecks, or downstream Pulsar cluster performance problems that could lead to replication lag.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 321, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing User Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of active user transactions in TiDB. Critical for identifying long-running transactions that may block garbage collection (GC) and cause performance degradation or storage bloat.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10015, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the latency of write operations to cloud storage in TiCDC's cloud storage sink. Critical for diagnosing performance bottlenecks when replicating data to cloud storage destinations like S3 or GCS, helping identify slow writes that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5994, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of read I/O throughput in TiKV instances, broken down by operation type. Essential for diagnosing read performance bottlenecks, identifying which storage operations consume the most I/O bandwidth, and detecting abnormal disk read patterns that may impact query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[45s]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14914, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Type", "group": "Backup Log", "used_for": "Used for monitoring the rate of different internal message types processed by TiKV's log backup actor. Helps diagnose log backup performance issues by identifying which message types are most frequently processed or experiencing bottlenecks during backup operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_interal_actor_acting_duration_sec_count{instance=~\"$instance\"}[$__rate_interval])) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 182, "dashboard": "TiDB-Cluster-Clinic", "title": "Push Task Duration 95", "group": "GC", "used_for": "Used for monitoring the 95th percentile latency of pushing range tasks from TiDB to TiKV during garbage collection operations. Critical for diagnosing GC performance bottlenecks and identifying instances experiencing delays in distributing GC workload to TiKV nodes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 24, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk iops", "group": "Disk", "used_for": "Used for monitoring disk I/O operations per second (IOPS) on dedicated dataplane nodes, tracking both read and write operations. Essential for identifying disk bottlenecks, understanding storage workload patterns, and diagnosing performance degradation caused by excessive disk activity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 91, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk iops", "group": "Disk", "used_for": "Used for monitoring write I/O operations per second on disk devices across dataplane nodes. Essential for identifying disk write bottlenecks, detecting abnormal write patterns, and correlating storage performance issues with workload changes.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 197, "dashboard": "TiDB-Cluster-Clinic", "title": "Acquire Pessimistic Locks Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the latency of acquiring pessimistic locks during transactions at p80, p95, and p99 percentiles. Critical for diagnosing transaction performance issues, lock contention problems, and identifying when pessimistic locking operations are causing delays in write-heavy workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10017, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Flush duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the time taken to flush data files to cloud storage in TiCDC's cloud storage sink. Critical for diagnosing slow data replication to cloud destinations and identifying potential bottlenecks in the file upload process.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 46, "dashboard": "PD-Cluster-Clinic", "title": "Scheduler is running", "group": "Scheduler", "used_for": "Used for monitoring which schedulers are currently active and allowed to run in PD. Essential for verifying scheduler configuration changes and troubleshooting why certain data rebalancing or region scheduling operations may not be occurring as expected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"allow\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 87, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader movement", "group": "Scheduler", "used_for": "Used for monitoring the movement of Raft leaders between TiKV stores during balance operations. Helps diagnose leader distribution issues and verify that PD's balance-leader scheduler is actively redistributing leaders to achieve cluster load balancing.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5995, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO threshold", "group": "IO Breakdown", "used_for": "Used for monitoring the configured I/O rate limits (bandwidth thresholds) in TiKV's rate limiter by operation type. Helps administrators verify I/O throttling settings and diagnose whether performance issues are caused by hitting configured bandwidth limits.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_rate_limiter_max_bytes_per_sec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average wait time per disk I/O operation (both reads and writes) on dedicated dataplane nodes. Essential for identifying disk performance bottlenecks and diagnosing storage-related latency issues that may impact TiDB cluster performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 7225, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate Limiter Request Wait Duration", "group": "IO Breakdown", "used_for": "Used for monitoring the 99th percentile wait duration of requests throttled by TiKV's rate limiter. Critical for diagnosing when storage I/O throttling is causing request delays and identifying performance bottlenecks due to resource contention or quota limits.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "rate(tikv_rate_limiter_request_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_rate_limiter_request_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10013, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in the Pulsar client producer queue for TiCDC changefeeds. Helps identify message backlog issues and potential bottlenecks in data replication to Pulsar, indicating when the producer cannot keep up with the change data capture rate.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 94, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average time spent waiting for disk write operations to complete on dataplane nodes. Essential for identifying disk I/O bottlenecks and diagnosing storage performance degradation that could impact TiKV write throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 3653, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob file synchronization operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing Titan write performance issues and identifying when large value storage is experiencing fsync bottlenecks that could impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_average\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile95\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile99\"})", "max(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 195, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Lifetime Reach Counter", "group": "Transaction", "used_for": "Used for monitoring the rate at which transaction TTL (Time To Live) lifetimes are reached in TiDB. Helps identify transactions that are holding locks for too long, which can indicate potential lock contention issues, slow transactions, or applications not properly committing/rolling back transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10019, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the busy ratio of cloud storage sink workers in TiCDC, indicating how much time workers spend actively processing data versus idle time. Essential for identifying worker saturation issues and determining if additional worker capacity is needed to handle replication workload to cloud storage.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])/10) by (namespace,changefeed,id,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3651, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file sync operations in TiKV's Titan storage engine. Essential for diagnosing I/O performance issues and understanding the frequency of data persistence operations when using Titan for large value storage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14912, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile latency of internal message handling in TiKV's backup log subsystem. Critical for diagnosing performance bottlenecks in the log backup actor system, helping identify which message types are experiencing slow processing that could impact backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance" ], "promql": [ "sum(histogram_quantile(0.99, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5018, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC action", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of Titan blob garbage collection operations by action type in TiKV's storage engine. Essential for tracking blob file cleanup activity and diagnosing issues with Titan's space reclamation process when storage usage is unexpectedly high.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_action_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's balance leader scheduler, which redistributes region leaders across TiKV nodes to balance load. Essential for diagnosing leader distribution imbalances and verifying that the scheduler is actively working to equalize leadership across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 571, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the replication lag between TiCDC's changefeed checkpoint and the upstream TiDB cluster's current timestamp. Critical for identifying replication delays, ensuring data freshness in downstream systems, and diagnosing changefeed performance issues or bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572468, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn Scheduler Pool Wait Duration", "group": "Scheduler", "used_for": "Used for monitoring the wait time distribution of transactions in TiKV's scheduler worker pool before being processed. Critical for diagnosing transaction processing bottlenecks and identifying when the scheduler is overloaded, causing transactions to queue excessively before execution.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched-worker.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring disk I/O bandwidth (read and write rates) on dedicated dataplane nodes in a Kubernetes cluster. Essential for identifying disk bottlenecks, detecting abnormal I/O patterns, and diagnosing performance issues related to storage throughput.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14913, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile latency of internal message processing in TiKV's backup log subsystem. Helps diagnose performance bottlenecks in the log backup actor system by identifying which message types are experiencing high processing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(histogram_quantile(0.9, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 309, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing Internal Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of TiDB's internal transactions, such as those used for GC (garbage collection) and statistics updates. Critical for diagnosing when internal maintenance operations are taking too long and potentially impacting cluster performance or blocking user transactions.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"internal\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 86, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region movement", "group": "Scheduler", "used_for": "Used for monitoring the movement of regions during balance operations in PD's region scheduler. Helps diagnose load balancing effectiveness by showing which stores are sources (losing regions) and targets (gaining regions), essential for identifying uneven data distribution and scheduler behavior issues.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 83, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Safepoint OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of successful safepoint load operations per TiDB instance. Helps diagnose transaction GC (garbage collection) coordination issues and identify instances having trouble retrieving the cluster's safepoint for MVCC cleanup.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ok\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14271, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Throughput ", "group": "Backup Log", "used_for": "Used for monitoring the data read throughput from RocksDB during the initial scan phase of TiKV log backup operations. Essential for diagnosing backup performance issues and understanding how quickly TiKV can scan and read existing data when starting a new backup task.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op=~\"read_bytes\"}[$__rate_interval])) BY (op, cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13132, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the p99 and p999 latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying tail latency issues that may impact transaction commit times and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 203, "dashboard": "TiDB-Cluster-Clinic", "title": "No Available Connection Counter", "group": "Batch Client", "used_for": "Used for monitoring connection unavailability in TiDB's batch client when attempting to communicate with TiKV. Helps diagnose connection pool exhaustion or network connectivity issues that prevent TiDB from establishing connections to TiKV nodes.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 199, "dashboard": "TiDB-Cluster-Clinic", "title": "Pessimistic Statement Retry OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of pessimistic transaction statement retries in TiDB. Helps identify transaction conflicts and contention issues that cause statements to retry, which can indicate lock conflicts or write-write conflicts affecting transaction throughput.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 589, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Scheduler", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring near real-time replication to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3649, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC keys flow", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the rate of key data flow during Titan blob garbage collection operations in TiKV. Helps diagnose Titan GC efficiency and identify potential issues with blob file cleanup that could impact storage space reclamation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 93, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring the write bandwidth (bytes per second) to physical disks on dedicated dataplane nodes. Essential for identifying disk I/O bottlenecks, detecting write-heavy workloads, and diagnosing storage performance issues that may impact TiKV write operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2224, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks duration", "group": "GC", "used_for": "Used for monitoring the P95, P99, and maximum duration of garbage collection tasks in TiKV. Essential for diagnosing GC performance issues that could impact cluster stability, identifying slow GC operations, and ensuring old MVCC versions are being cleaned up efficiently without causing latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3410, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the garbage collection latency in TiKV's Titan storage engine, which manages large values stored separately from RocksDB. Critical for diagnosing performance issues related to Titan blob file cleanup, especially when experiencing write amplification or storage reclamation delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_average\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile95\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile99\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 53, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region scheduler", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the activity rate of PD's balance-region scheduler, which redistributes regions across TiKV stores to achieve balanced load. Essential for diagnosing uneven data distribution issues and verifying that region balancing operations are occurring as expected during cluster rebalancing or after topology changes.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 32, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of individual disk read and write operations per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and potential bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 180, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Client Unavailable Duration 95", "group": "Batch Client", "used_for": "Used for monitoring the 95th percentile duration when TiKV batch clients are unavailable in TiDB. Critical for diagnosing connection issues between TiDB and TiKV, helping identify network problems or TiKV overload that causes batch request failures.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14270, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Operation ", "group": "Backup Log", "used_for": "Used for monitoring the rate of RocksDB operations during log backup's initial scan phase in TiKV. Helps diagnose performance bottlenecks and operation patterns when backup starts scanning existing data, excluding read byte metrics to focus on operation counts across different column families.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op!~\"read_bytes\"}[$__rate_interval])) BY (op, cf) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13257, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft store thread operations in TiKV, tracking p99, p999, and average durations. Critical for diagnosing Raft consensus bottlenecks and identifying when the store thread becomes a performance bottleneck in the write path.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 212, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Types Per Second", "group": "Transaction", "used_for": "Used for monitoring the rate of different transaction commit types per second in TiDB, including standard two-phase commit, async commit, and one-phase commit (1PC) transactions. Essential for understanding transaction workload patterns and identifying which commit protocols are being used, helping diagnose transaction performance issues and validate optimization strategies like async commit adoption.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1458, "dashboard": "PD-Cluster-Clinic", "title": "Balance Hot Region scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's hot region balance scheduler, which automatically redistributes regions with high I/O load across TiKV nodes. Essential for diagnosing uneven I/O distribution issues and verifying whether the scheduler is actively balancing hot spots in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 546, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table replication state", "group": "Scheduler", "used_for": "Used for monitoring the current replication state of tables being synchronized by TiCDC changefeeds. Helps identify tables that are stuck, stopped, or experiencing replication issues, enabling quick diagnosis of data synchronization problems in CDC pipelines.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13259, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of applying Raft log entries to TiKV's state machine, tracking p99, p999, and average durations. Critical for diagnosing write path performance issues and identifying bottlenecks in the Raft apply process that can impact overall write throughput and latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_apply_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 121, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection tasks in TiKV, including successful GC operations, skipped tasks, and failed tasks by type. Essential for diagnosing GC performance issues, identifying task failures, and understanding why certain GC operations are being skipped, which can impact storage space reclamation and MVCC version cleanup.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_tasks_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_storage_gc_skipped_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_gcworker_gc_task_fail_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_gc_worker_too_busy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 969, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC seconds", "group": "GC", "used_for": "Used for monitoring the maximum (100th percentile) duration of garbage collection operations initiated by TiDB to TiKV. Critical for identifying GC performance bottlenecks that could impact cluster stability and transaction visibility, especially when GC operations take excessively long to complete.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance, le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3340, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC bytes flow", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the byte flow rate during Titan blob garbage collection operations in TiKV. Helps diagnose Titan storage engine performance by tracking how much data is being processed during blob GC, which is critical for understanding storage reclamation efficiency and identifying potential I/O bottlenecks in large value workloads.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 549, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks accept", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduling tasks being accepted by TiCDC changefeeds. Helps identify scheduling bottlenecks and verify that the TiCDC scheduler is properly distributing replication tasks across workers.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 224, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Commit .99 Backoff", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile backoff duration during transaction commits in TiDB. Critical for diagnosing transaction commit delays caused by lock conflicts, region unavailability, or other transient issues that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 204, "dashboard": "TiDB-Cluster-Clinic", "title": "Wait Connection Establish Duration", "group": "Batch Client", "used_for": "Used for monitoring the 99.99th percentile latency of establishing batch client connections between TiDB and TiKV. Critical for diagnosing connection pool issues, network problems, or TiKV overload that causes extreme delays in connection establishment.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1459, "dashboard": "PD-Cluster-Clinic", "title": "Split Bucket scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's split-bucket scheduler, which automatically splits hot Regions into smaller buckets to improve load balancing. Essential for diagnosing uneven data distribution issues and verifying that the scheduler is actively working to distribute hotspot workloads across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"split-bucket-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 228, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Receive Average Duration", "group": "Batch Client", "used_for": "Used for monitoring the average and P99 latency of batch receive operations between TiDB and TiKV clients. Critical for diagnosing network communication delays and identifying instances with slow batch RPC response times that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14915, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Trigger Reason", "group": "Backup Log", "used_for": "Used for tracking the reasons that trigger initial scanning operations in TiKV's backup log feature. Helps diagnose why backup scans are initiated (e.g., new region, leader transfer, or configuration changes) and identify potential issues causing excessive or unexpected scan triggers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(increase(tikv_log_backup_initial_scan_reason{instance=~\"$instance\"}[$__rate_interval])) by (reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 548, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks", "group": "Scheduler", "used_for": "Used for monitoring the distribution and count of scheduling tasks across different schedulers in TiCDC changefeeds. Helps identify scheduling imbalances, task backlogs, or scheduler bottlenecks that could impact data replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 15176, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Key Putting", "group": "Backup Log", "used_for": "Used for monitoring the rate of region checkpoint metadata key operations during TiKV log backup. Helps diagnose backup performance issues and identify bottlenecks in checkpoint metadata persistence, which is critical for tracking backup progress and ensuring data consistency during point-in-time recovery operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_metadata_key_operation{instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 966, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC worker actions", "group": "GC", "used_for": "Used for monitoring the rate of different garbage collection (GC) worker actions in TiDB, broken down by action type. Essential for diagnosing GC performance issues, identifying bottlenecks in the GC process, and understanding which GC operations are consuming the most resources.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read (rrqm) and write (wrqm) requests per second on disk devices in the dataplane node. Helps assess I/O efficiency and identify whether the kernel is successfully merging adjacent I/O requests to reduce disk operations and improve performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 226, "dashboard": "TiDB-Cluster-Clinic", "title": "SafeTS Update Conuter", "group": "Transaction", "used_for": "Used for monitoring the rate of SafeTS (Safe Timestamp) updates between TiDB and TiKV stores, broken down by result status and store. Essential for diagnosing transaction coordination issues, stale read problems, and detecting communication failures between TiDB and TiKV that could impact transaction consistency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 92, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of disk I/O operations (both read and write) per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and potential bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13263, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store batch wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft store threads spend waiting to batch operations before processing, tracking p99, p999, and average latencies. Critical for diagnosing Raft processing bottlenecks and identifying when store threads are delayed due to batching inefficiencies or thread contention.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4021, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC input file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size of input files during Titan blob garbage collection operations in TiKV. Helps diagnose Titan storage engine GC efficiency and identify abnormal blob file sizes that may impact GC performance and storage reclamation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_average\"})", "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile95\"})", "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile99\"})", "max(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 588, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks running", "group": "Scheduler", "used_for": "Used for monitoring the number of actively running scheduler tasks in TiCDC changefeeds. Helps identify scheduling bottlenecks, task accumulation issues, and abnormal task execution patterns that could impact data replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13261, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store propose wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft proposals spend waiting in the store thread queue before being processed in TiKV. Critical for diagnosing write latency issues caused by store thread congestion or scheduling delays, especially when p99/p999 values are high, indicating that requests are queuing excessively before Raft processing begins.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4022, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC output file size", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the size of output files generated during Titan blob garbage collection in TiKV. Helps assess GC efficiency and identify potential issues with blob file compaction, where unusually large or small output files may indicate suboptimal storage reclamation or fragmentation problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_average\"})", "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile95\"})", "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile99\"})", "max(tikv_engine_blob_gc_outputt_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 225, "dashboard": "TiDB-Cluster-Clinic", "title": "Max SafeTS Gap", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the maximum gap between SafeTS (safe timestamp) and current time in TiDB's transaction system. Critical for diagnosing stale read performance issues and identifying potential delays in timestamp synchronization that could affect transaction consistency and read latency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2823, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "ResolveLocks Progress", "group": "GC", "used_for": "Used for monitoring the progress and results of lock resolution operations in TiKV during garbage collection. Helps diagnose issues with transaction cleanup and identify when locks cannot be resolved, which can block GC and cause storage bloat.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"resolve-locks.*\"}) by (result)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15716, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of backup log advancer tick operations in TiKV. Helps diagnose performance issues with the log backup checkpoint advancement process, which is critical for ensuring timely and consistent point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "tidb_cluster" ], "promql": [ "sum(increase(tidb_log_backup_advancer_tick_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", step=~\"tick\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15544, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Checkpoint Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of batch sizes when TiKV processes checkpoint requests during log backup operations. Helps diagnose backup efficiency issues by revealing whether checkpoints are being processed in optimal batch sizes or if batching configuration needs adjustment.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_advancer_batch_size_bucket{type=\"checkpoint\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 108, "dashboard": "PD-Cluster-Clinic", "title": "Balance Direction", "group": "Scheduler", "used_for": "Used for monitoring the direction and volume of region balance operations in PD's scheduler. Helps diagnose load balancing behavior by showing which stores are sources or targets of region transfers, essential for troubleshooting uneven data distribution and identifying potential hotspots or storage imbalances.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, source, target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1424, "dashboard": "PD-Cluster-Clinic", "title": "Patrol Region time", "group": "Scheduler", "used_for": "Used for monitoring the time taken by PD to patrol and check all regions in the cluster. Essential for diagnosing region health check performance and identifying when region scanning becomes slow, which can delay detection of region issues like unavailability or imbalanced distribution.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} != 0" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 95, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read and write requests per second on disk devices in dedicated dataplane nodes. Helps identify disk I/O optimization effectiveness and potential bottlenecks when the OS merges adjacent I/O requests to improve throughput.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 141, "dashboard": "PD-Cluster-Clinic", "title": "Rule checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of rule checker events in PD's scheduler, broken down by rule name. Helps diagnose placement rule violations and scheduling issues related to replica placement policies, label constraints, and isolation requirements.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"rule_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13265, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send to write queue duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft store thread messages to the write queue in TiKV, tracking p99, p999, and average durations. Critical for diagnosing bottlenecks in the Raft store thread's message dispatching pipeline and identifying when the write queue becomes a performance constraint.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the percentage of used storage across different devices. Essential for preventing disk full scenarios and identifying nodes approaching storage capacity limits that could impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "((max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) /\n max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) * 100" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 3344, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file garbage collection operations in TiKV's Titan storage engine. Essential for tracking Titan GC activity and diagnosing issues with blob file cleanup that could lead to disk space accumulation or storage inefficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 231, "dashboard": "TiDB-Cluster-Clinic", "title": "Assertion", "group": "Transaction", "used_for": "Used for monitoring the rate of prewrite assertion operations in TiDB transactions, categorized by assertion type. Helps diagnose transaction conflicts and optimistic locking issues that can impact transaction latency and throughput.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 545, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table count", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the total number of tables being replicated by TiCDC changefeeds. Helps identify changefeed scale, verify table distribution across the scheduler, and diagnose issues when tables are unexpectedly added or removed from replication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2821, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC Progress", "group": "GC", "used_for": "Used for monitoring the progress of TiKV's automatic garbage collection process by tracking the ratio of scanned regions to total regions per instance. Helps operators verify GC is progressing normally across the cluster and identify instances where GC may be stuck or lagging behind.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_gcworker_autogc_processed_regions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572666, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Failure Reason", "group": "Backup Log", "used_for": "Used for tracking the reasons why region checkpoint operations fail during log backup in TiKV, excluding retryable scan errors. Essential for diagnosing log backup reliability issues and identifying specific failure patterns that may require intervention or configuration adjustments.", "component": "TIKV", "phenomenon": "error_rate", "object": "region", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_region_request_failure{reason!=\"retryable-scan-region\"}[$__interval])) by (reason)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 49, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Used", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the difference between total filesystem size and available space. Essential for identifying storage capacity issues and preventing disk full scenarios that could impact node operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "(max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 547, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture tables count", "group": "Scheduler", "used_for": "Used for monitoring the number of tables assigned to each TiCDC capture node across changefeeds. Helps identify table distribution imbalances in the scheduler and diagnose potential workload skew issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 70, "dashboard": "PD-Cluster-Clinic", "title": "Replica checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of replica checker events in PD's scheduler. Helps diagnose replica placement issues, such as missing replicas, incorrect replica counts, or replica distribution problems that trigger the checker to take corrective actions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"replica_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2589, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC speed", "group": "GC", "used_for": "Used for monitoring the rate at which TiKV's garbage collection (GC) deletes obsolete MVCC versions. Essential for verifying GC efficiency and diagnosing issues when storage space isn't being reclaimed as expected or when GC is impacting cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 305, "dashboard": "TiDB-Cluster-Clinic", "title": "Ignore Event Per Minute", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for tracking the rate of SQL statements ignored by TopSQL feature per minute, categorized by ignore reason type. Helps diagnose why certain queries are not being captured in TopSQL statistics, which is critical when troubleshooting missing query data or understanding TopSQL sampling behavior.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_ignored_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 51, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiKV and TiFlash persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized deployments.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2822, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC SafePoint", "group": "GC", "used_for": "Used for monitoring the automatic garbage collection (GC) safe point timestamp in TiKV, which indicates the oldest data version that must be retained. Essential for diagnosing GC progress issues, MVCC version accumulation problems, and ensuring transactions can access historical data within the configured retention period.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_gcworker_autogc_safe_point{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (instance) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the P99, P90, and P80 latency of different transaction execution states in TiDB. Essential for diagnosing transaction performance bottlenecks by identifying which transaction phases (idle, executing, committing, etc.) are experiencing high latency and causing overall transaction slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 15359, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile duration of backup log advancement ticks in TiKV. Critical for diagnosing performance issues in the log backup process, particularly when backup operations are slow or experiencing delays in advancing the backup checkpoint.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [], "promql": [ "sum(histogram_quantile(0.99, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572665, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Result", "group": "Backup Log", "used_for": "Used for monitoring the results of log backup region requests in TiKV, categorized by success or failure status. Helps diagnose log backup reliability issues and identify request failures that may impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_region_request[$__interval])) by (result)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 71, "dashboard": "PD-Cluster-Clinic", "title": "Region merge checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of region merge operations checked by PD's merge checker. Essential for diagnosing region merge performance issues, identifying merge bottlenecks, and understanding whether regions are being consolidated efficiently to reduce cluster overhead.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"merge_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 253, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction with lock execution states duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution (p80, p90, p99) of transaction execution states that hold locks in TiDB. Critical for diagnosing lock contention issues and identifying transactions that may be blocking others, helping pinpoint the root cause of transaction performance degradation and deadlock scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572857, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send proposal duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft proposals from TiKV store threads to Raft threads, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying when the store thread is slow in dispatching proposals to the Raft state machine.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 543, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Scheduler", "used_for": "Used for identifying which table ID is experiencing the slowest replication performance in TiCDC's scheduler. Helps pinpoint specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 210, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction write sizes in TiDB, showing how many bytes each transaction writes to TiKV. Essential for identifying large transactions that may cause performance bottlenecks, memory pressure, or timeout issues during commit.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 239, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Report Duration", "group": "TopSQL", "used_for": "Used for monitoring the 99th percentile latency of TopSQL report generation and submission in TiDB and TiKV. Critical for diagnosing performance issues with the TopSQL feature itself, helping identify when report collection overhead becomes excessive or when communication delays occur between components.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[5m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 13269, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write kv db end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of completing KV database writes in TiKV's Raft store thread, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying when data persistence to RocksDB is experiencing delays in the Raft replication waterfall.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2991, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Thread CPU", "group": "Pessimistic Locking", "used_for": "Used for monitoring CPU usage of TiKV's lock manager threads, including waiter manager and deadlock detector threads. Critical for diagnosing performance issues in pessimistic locking scenarios, especially when experiencing lock contention, deadlock detection overhead, or transaction conflicts that cause high CPU consumption in lock management.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"waiter_manager.*\"}[2m])) by (instance, name)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"deadlock_detect.*\"}[2m])) by (instance, name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2877, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Handled tasks", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of different task types handled by TiKV's lock manager in pessimistic transaction mode. Essential for diagnosing lock contention issues, deadlock detection performance, and understanding the distribution of lock-related operations across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_task_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 544, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table replication state", "group": "Scheduler", "used_for": "Used for identifying tables with the slowest replication progress in TiCDC changefeeds. Critical for diagnosing replication lag issues and pinpointing which specific tables are causing bottlenecks in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 255, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the time spent in different transaction execution states (idle, running, committing, etc.) and overall transaction duration in TiDB. Essential for diagnosing transaction performance bottlenecks and identifying which execution phase is causing delays or slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_txn_state_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 13267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store before write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency before Raft log writes are persisted to disk in TiKV's store thread, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying delays in the Raft log replication pipeline before actual disk writes occur.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 243, "dashboard": "TiDB-Cluster-Clinic", "title": "Total Report Count", "group": "TopSQL", "used_for": "Used for monitoring the volume and success rate of TopSQL reports sent from TiDB and TiKV instances. Essential for diagnosing TopSQL feature health, identifying report delivery failures, and ensuring query performance data is being properly collected and transmitted for analysis.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)", "sum(increase(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, result)", "sum(tikv_resource_metering_report_data_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"sent\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 109, "dashboard": "PD-Cluster-Clinic", "title": "Filter source", "group": "Scheduler", "used_for": "Used for monitoring the frequency and reasons why PD's scheduler filters out source stores when creating operator schedules. Helps diagnose why certain stores are being excluded from scheduling operations, which is critical for troubleshooting unbalanced region distribution and scheduling bottlenecks.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[2m])) by (source, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 28, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC interval", "group": "GC", "used_for": "Used for monitoring the configured garbage collection (GC) interval in TiKV. Helps verify GC scheduling frequency and diagnose issues when old MVCC versions aren't being cleaned up as expected, which can lead to storage bloat and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"tikv_gc_run_interval\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC lifetime", "group": "GC", "used_for": "Used for monitoring the configured GC (Garbage Collection) lifetime setting in TiKV, which determines how long historical MVCC versions are retained. Essential for diagnosing issues related to data retention policies, storage space usage, and ensuring the GC lifetime is appropriately configured to balance between historical data access needs and storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"tikv_gc_life_time\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13273, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of persisting Raft store metadata to disk in TiKV, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and disk I/O performance issues that affect Raft log persistence and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 97, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiDB persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized TiDB deployments.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 241, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Profiling OPS", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CPU profiling operations in TiDB instances. Helps verify that CPU profiling is actively collecting performance data for TopSQL analysis and troubleshooting query performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_cpu_profile_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Waiter lifetime duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring how long transactions wait for locks to be released in TiKV's pessimistic locking system, tracking average, p99, and p9999 latencies. Critical for diagnosing lock contention issues and identifying transactions experiencing excessive wait times due to conflicting pessimistic locks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 254, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction enter state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions enter different states (e.g., idle, running, committed) in TiDB. Helps diagnose transaction lifecycle issues, identify abnormal state transitions, and understand transaction processing patterns during performance troubleshooting.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 15360, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile duration of backup log advancement ticks in TiKV. Helps diagnose performance issues in the log backup process by identifying when backup operations are taking longer than expected, which could indicate storage bottlenecks or resource contention affecting backup efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "sum(histogram_quantile(0.9, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption on TiKV and TiFlash persistent volume claims in Kubernetes environments. Essential for tracking storage utilization and preventing disk full issues that could cause data write failures or service disruptions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 6596, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC in Compaction Filter", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection operations performed during RocksDB compaction in TiKV. Helps diagnose GC efficiency by tracking filtered keys, skipped operations, and MVCC rollback events, which is essential for understanding storage cleanup performance and identifying potential GC bottlenecks.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gc_compaction_filtered{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_skip{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_mvcc_rollback{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_orphan_versions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_perform{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode,type)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_met{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_handled{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_wasted{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 541, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table checkpoint", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for identifying the slowest table's replication checkpoint timestamp in TiCDC changefeeds compared to the current PD TSO. Critical for detecting replication lag bottlenecks caused by specific tables that are falling behind, helping pinpoint which tables are slowing down the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 242, "dashboard": "TiDB-Cluster-Clinic", "title": "Report Data Count Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the volume of Top SQL data being reported by TiDB instances per minute, broken down by data type. Helps diagnose Top SQL feature overhead and identify potential issues with SQL statistics collection or reporting bottlenecks that could impact query performance monitoring.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_report_data_total_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, job)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4018, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Waiting Queue", "group": "Pessimistic Locking", "used_for": "Used for monitoring the size and status of pessimistic lock waiting queues in TiKV. Essential for diagnosing lock contention issues, identifying transactions waiting for locks, and detecting potential deadlock situations that impact transaction throughput.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(tikv_lock_manager_wait_table_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)", "sum(max_over_time(tikv_lock_wait_queue_entries_gauge_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 245, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV Stat Task OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of resource metering statistics tasks processed by TiKV instances. Helps diagnose TopSQL feature performance and identify issues with resource consumption tracking when analyzing query-level resource usage patterns.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_resource_metering_stat_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 96, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption of TiDB Persistent Volume Claims (PVCs) in Kubernetes environments. Essential for tracking storage utilization and preventing disk space exhaustion issues that could impact TiDB server operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 110, "dashboard": "PD-Cluster-Clinic", "title": "Filter target", "group": "Scheduler", "used_for": "Used for monitoring how often PD's scheduler filters out potential target stores during region scheduling operations. Helps diagnose why certain stores are being excluded as scheduling targets, revealing issues with placement rules, label constraints, or store state problems that prevent optimal region distribution.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[2m])) by (target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"distinct-filter\"}[2m])) by (source, target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[2m])) by (source, target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[2m])) by (source, target, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13277, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit but not persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time gap between when Raft logs are committed and when they are actually persisted to disk in TiKV store threads. Critical for diagnosing write pipeline bottlenecks and identifying delays in the Raft log persistence process that could impact data durability and write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8767, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan write details", "group": "GC", "used_for": "Used for monitoring the rate of keys scanned during garbage collection in TiKV's write column family, broken down by key mode and tag. Essential for diagnosing GC performance issues, understanding GC workload patterns, and identifying potential bottlenecks in the write CF cleanup process.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\", cf=\"write\"}[2m])) by (key_mode,tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572734, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Try Advance Trigger Time", "group": "Backup Log", "used_for": "Used for monitoring the frequency of log backup advancement attempts in TiKV. Helps diagnose backup progress issues and identify whether the log backup advancer is actively triggering checkpoint advancement operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__interval" ], "promql": [ "increase(tidb_log_backup_advancer_tick_duration_sec_count{step=\"try-advance\"}[$__interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 256, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction leave state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions exit different states in TiDB, categorized by transaction type. Helps diagnose transaction lifecycle issues, identify abnormal transaction patterns, and troubleshoot problems related to transaction commit, rollback, or timeout behaviors.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 590, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table resolved ts", "group": "Scheduler", "used_for": "Used for monitoring the resolved timestamp of the slowest table in TiCDC changefeeds, compared against the cluster's current TSO. Critical for identifying replication lag bottlenecks caused by specific slow tables that are holding back the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13271, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the duration of the store write end phase in TiKV's Raft replication waterfall, tracking p99, p999, and average latencies. Critical for diagnosing write performance bottlenecks in the final stage of Raft log persistence, helping identify when store-level write operations are experiencing delays that impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance", "used_for": "Used for monitoring when AWS EBS volumes exceed their provisioned IOPS or throughput limits on dedicated dataplane nodes. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to I/O throttling and degraded database performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2995, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detect duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of deadlock detection in TiKV's pessimistic locking mechanism, showing both average and p99 durations. Critical for diagnosing transaction contention issues and identifying when deadlock detection becomes a performance bottleneck in high-concurrency pessimistic transaction workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_detect_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_detect_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572733, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get Region Operation Count", "group": "Backup Log", "used_for": "Used for monitoring the frequency of region retrieval operations during TiDB log backup processes. Helps diagnose backup performance issues by tracking how often the backup advancer queries region information, which can indicate region scanning bottlenecks or excessive region fragmentation impacting backup efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(tidb_log_backup_advancer_tick_duration_sec_count{step=\"get-regions-in-range\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 295, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Usage", "group": "Transaction", "used_for": "Used for monitoring the usage patterns of TiDB's fair locking mechanism in transactions, categorized by lock type. Helps diagnose transaction contention issues and understand how frequently fair locking is being triggered to prevent lock starvation in high-concurrency scenarios.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_session_transaction_fair_locking_usage{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8768, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan default details", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection key scanning operations in TiKV's default column family, broken down by key mode and tag. Helps diagnose GC performance issues and understand the volume of keys being processed during garbage collection cycles.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\", cf=\"default\"}[2m])) by (key_mode,tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Detect error", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of errors occurring in TiKV's pessimistic lock manager, categorized by error type. Critical for diagnosing transaction conflicts, deadlocks, and other locking-related issues that may impact transaction success rates and application performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13275, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit and persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the p99 and p999 latency of Raft log commit and persist operations in TiKV's store thread. Critical for diagnosing write path bottlenecks and identifying when disk I/O or Raft replication is causing slow writes to storage.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 560, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "Scheduler", "used_for": "Used for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scan activity patterns and diagnose potential bottlenecks in TiCDC's data replication process when capturing changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1439, "dashboard": "PD-Cluster-Clinic", "title": "scatter operator event", "group": "Scatter and Splitter", "used_for": "Used for monitoring the outcome of scatter operations in PD, tracking successful, failed, and skipped scatter operator events. Essential for diagnosing region distribution issues and identifying problems with load balancing operations when regions fail to scatter across TiKV stores as expected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"skip\"}[2m])) by (event)", "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fail\"}[2m])", "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"success\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 4019, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detector leader", "group": "Pessimistic Locking", "used_for": "Used for identifying which TiKV instance is currently serving as the deadlock detector leader in pessimistic locking scenarios. Essential for troubleshooting deadlock detection issues and verifying proper leader election when investigating transaction conflicts or lock-related problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1435, "dashboard": "PD-Cluster-Clinic", "title": "scatter store selection", "group": "Scatter and Splitter", "used_for": "Used for monitoring how PD distributes scattered regions across TiKV stores, showing both leader and follower replica placement. Essential for diagnosing load balancing issues during table scatter operations and identifying whether regions are being evenly distributed across stores.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"false\"}[2m])) by (store)", "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"true\"}[2m])) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 297, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB CPU Usage", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiDB instances in the TTL (Time To Live) context, comparing actual CPU usage rate against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks during TTL operations and determining if TiDB instances need resource scaling to handle TTL workload efficiently.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 261, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction state count change", "group": "Transaction", "used_for": "Used for tracking the rate of change in transaction state transitions within TiDB, comparing state duration counts against state entry counts. Helps identify transaction state management anomalies, such as transactions getting stuck in certain states or abnormal state transition patterns that could indicate deadlocks or resource contention issues.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "", "used_for": "Used for monitoring CPU usage of TiKV backup-related threads, including backup workers and backup endpoints. Essential for diagnosing backup performance issues, identifying CPU bottlenecks during backup operations, and ensuring backup tasks don't consume excessive resources that could impact cluster performance.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 301, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV IO MBps", "group": "TTL", "used_for": "Used for monitoring TiKV's I/O throughput in megabytes per second during TTL (Time To Live) operations, tracking WAL writes and data reads across instances. Essential for identifying I/O bottlenecks and uneven load distribution when TTL background tasks are deleting expired data, helping diagnose performance degradation caused by excessive disk I/O.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 296, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Keys", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB's pessimistic transaction model, categorized by locking type. Helps diagnose lock contention issues and identify when transactions are aggressively acquiring locks, which can indicate hotspot problems or inefficient transaction patterns affecting cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 691, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table region count", "group": "Slow Table", "used_for": "Used for monitoring the number of regions in the slowest table being replicated by TiCDC changefeeds. Helps identify replication bottlenecks caused by tables with excessive region counts, which can impact overall CDC performance and lag.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 689, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Slow Table", "used_for": "Used for identifying which table ID is experiencing the slowest replication in TiCDC changefeeds. Essential for pinpointing specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 35, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Thread Count", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the number of backup threads allocated in each TiKV instance's backup thread pool. Essential for diagnosing backup performance issues and ensuring adequate thread resources are available during backup operations, especially when backups are slow or experiencing I/O bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 8, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Helps identify read-heavy workloads, diagnose storage performance bottlenecks, and verify that EBS volume IOPS capacity meets application demands.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 299, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV CPU", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiKV instances in the context of TTL (Time To Live) operations. Helps identify if TTL background tasks are causing excessive CPU consumption on specific TiKV nodes, which could impact overall cluster performance.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\".*tikv\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1, "dashboard": "PD-Cluster-Clinic", "title": "gRPC Completed commands rate", "group": "gRPC", "used_for": "Used for monitoring the rate of completed gRPC commands per method in PD (Placement Driver). Helps identify which gRPC operations are most frequently executed and detect anomalies in command throughput, useful for diagnosing PD communication bottlenecks or unusual client behavior.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Errors", "group": "", "used_for": "Used for monitoring the rate of backup operation errors in TiKV instances over 2-minute intervals. Essential for quickly detecting backup failures and diagnosing data protection issues that could impact disaster recovery capabilities.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 64, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring write input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Essential for identifying storage bottlenecks, evaluating whether workloads are approaching EBS IOPS limits, and diagnosing write-heavy performance issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763572093, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total pessimistic locks memory size", "group": "Pessimistic Locking", "used_for": "Used for monitoring the total memory consumed by pessimistic locks across TiKV instances. Critical for diagnosing memory pressure issues caused by long-running transactions or lock accumulation, and helps identify potential out-of-memory risks in pessimistic transaction scenarios.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "tidb_cluster" ], "promql": [ "tikv_pessimistic_lock_memory_size{tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13279, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Process ready duration", "group": "Raft IO", "used_for": "Used for monitoring the time Raft proposals spend in the ready state before being processed in TiKV. Critical for diagnosing Raft processing bottlenecks and identifying delays in the consensus protocol pipeline that can impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572094, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "In-memory pessimistic locking result", "group": "Pessimistic Locking", "used_for": "Used for monitoring the success and failure rates of in-memory pessimistic lock operations in TiKV. Helps diagnose lock contention issues and memory-based locking performance, particularly useful when investigating transaction conflicts or pessimistic locking bottlenecks.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_in_memory_pessimistic_locking{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent between TiKV nodes. Essential for diagnosing data replication issues, identifying excessive snapshot transfers that may indicate slow followers or network problems, and understanding cluster recovery activities.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13281, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Process ready duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile duration of Raft log entries becoming ready for processing in TiKV, broken down by server instance. Critical for diagnosing Raft replication delays and identifying specific TiKV nodes experiencing slow consensus processing that could impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "PD-Cluster-Clinic", "title": "gRPC 99% Completed commands duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC command completion in PD servers. Critical for identifying slow gRPC operations and diagnosing communication bottlenecks between PD and other TiDB components (TiKV, TiDB servers).", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (grpc_method, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 38, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots by type (sending/receiving/applying) in TiKV. Essential for diagnosing region replication issues, identifying snapshot transfer bottlenecks, and understanding cluster data synchronization patterns during node recovery or scaling operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13283, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write loop duration", "group": "Raft IO", "used_for": "Used for monitoring the time distribution of TiKV's store write loop operations, which handles persisting Raft log entries to disk. Critical for diagnosing write path bottlenecks and identifying when the store thread is experiencing delays in processing Raft writes, which can impact overall cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL QPS By Type", "group": "TTL", "used_for": "Used for monitoring the queries per second (QPS) of TTL (Time To Live) operations in TiDB, broken down by SQL type and execution result. Helps identify TTL workload patterns, track the volume of expired data cleanup operations, and diagnose issues when TTL background jobs are running slower than expected or failing.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 37, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "", "used_for": "Used for monitoring the size distribution of SST files being backed up from TiKV's write column family. Helps identify backup I/O patterns and diagnose issues with backup performance or unusually large SST files during backup operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 37, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Snapshot generation wait duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile wait time before snapshot generation begins in TiKV's Raft store. Critical for diagnosing delays in snapshot creation that can impact region replication, leader transfers, and new replica bootstrapping performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573091, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pessimistic lock activities", "group": "Pessimistic Locking", "used_for": "Used for monitoring the number of transactions waiting in pessimistic lock queues, categorized by lock type. Essential for diagnosing lock contention issues, identifying bottlenecks in concurrent transaction processing, and understanding when transactions are blocked waiting for locks to be released.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_lock_wait_queue_entries_gauge_vec{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum read I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume read operations are experiencing high latency that could impact TiKV read performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 154, "dashboard": "TiDB-Cluster-Clinic", "title": "Compile Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL compilation latency in TiDB, broken down by SQL statement type. Critical for diagnosing slow query plan generation issues and identifying when the optimizer is taking excessive time to compile SQL statements before execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 156, "dashboard": "TiDB-Cluster-Clinic", "title": "Parse Duration", "group": "Executor", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile SQL parsing latency in TiDB, broken down by SQL statement type. Essential for identifying parsing bottlenecks when queries are slow, particularly for complex SQL statements or when experiencing high CPU usage during the parse phase.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 302, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows inserted per second by TiDB's Time To Live (TTL) feature. Helps identify TTL job activity levels and diagnose performance issues when TTL background tasks are inserting expired row records into system tables.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lengths of lock wait queues when transaction enqueues", "group": "Pessimistic Locking", "used_for": "Used for monitoring the distribution of lock wait queue lengths when transactions enqueue for pessimistic locks in TiKV. Helps diagnose lock contention issues and identify when multiple transactions are waiting for the same resources, which can indicate hotspot problems or inefficient transaction patterns.", "component": "TIKV", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_lock_wait_queue_length_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13285, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write loop duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of the Raft store write loop on each TiKV instance. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow Raft log persistence, which directly impacts transaction commit latency.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1461, "dashboard": "PD-Cluster-Clinic", "title": "HTTP Completed commands rate", "group": "HTTP", "used_for": "Used for monitoring the rate of completed HTTP commands handled by PD servers, broken down by service type and instance. Helps identify HTTP API throughput patterns, detect traffic anomalies, and diagnose potential bottlenecks in PD's HTTP service layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (service,ip)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 66, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum write I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume write operations are experiencing delays that could impact TiKV write performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 39, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency distribution of Raft log append operations in TiKV's storage engine. Critical for diagnosing write path performance issues, as slow log appends directly impact transaction commit latency and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 169, "dashboard": "TiDB-Cluster-Clinic", "title": "Execution Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile execution duration of SQL statements in TiDB, broken down by SQL type (SELECT, INSERT, UPDATE, etc.). Essential for identifying slow query execution patterns and diagnosing performance degradation in specific SQL operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 695, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table resolved ts", "group": "Slow Table", "used_for": "Used for monitoring the resolved timestamp progress of slow tables in TiCDC changefeeds compared to the current PD timestamp. Helps identify tables that are lagging behind in replication and diagnose which stage (scan/incremental) is causing the slowdown, critical for troubleshooting replication delays.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 693, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table checkpoint ts", "group": "Slow Table", "used_for": "Used for monitoring the checkpoint timestamp lag of slow tables in TiCDC changefeeds compared to the current PD timestamp. Helps identify replication delays caused by slow tables at different processing stages, enabling quick diagnosis of which tables are bottlenecking the data synchronization pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 699, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag details", "group": "Slow Table", "used_for": "Used for monitoring replication lag details of slow tables in TiCDC changefeeds across different processing stages (resolved TS, checkpoint TS, and TiKV min resolved TS). Critical for diagnosing which specific tables are causing replication delays and identifying bottlenecks in the CDC pipeline stages.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster", "tikv_instance" ], "promql": [ "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 67, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify I/O bottlenecks, diagnose storage performance issues, and verify whether read-intensive workloads are approaching EBS volume throughput limits.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "", "used_for": "Used for monitoring the rate at which TiKV generates SST (Sorted String Table) files during backup operations, measured in bytes per second. Essential for tracking backup performance, identifying slow backup processes, and ensuring backup operations meet throughput requirements.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 44, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Snapshot Duration", "group": "", "used_for": "Used for monitoring the duration of backup snapshot operations in TiKV. Essential for diagnosing slow backup performance and identifying bottlenecks during cluster backup processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 76, "dashboard": "TiDB-Cluster-Clinic", "title": "Expensive Executors OPS", "group": "Executor", "used_for": "Used for monitoring the rate of expensive executor operations in TiDB, categorized by executor type. Helps identify query patterns that trigger resource-intensive operations like table scans or complex joins, which can impact cluster performance and indicate missing indexes or inefficient queries.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 298, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of rows inserted by TTL (Time To Live) operations per hour in TiDB. Helps track TTL job activity and identify unusual insertion patterns or performance issues related to TTL background tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1h]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1462, "dashboard": "PD-Cluster-Clinic", "title": "HTTP 99% Completed commands duration", "group": "HTTP", "used_for": "Used for monitoring the 99th percentile latency of HTTP API commands handled by PD. Critical for identifying slow API responses and diagnosing performance issues in PD's HTTP service layer that may impact cluster management operations.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (service, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 44, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of snapshot sizes in TiKV, representing the largest snapshots being transferred during region replication or recovery. Critical for identifying abnormally large snapshots that could cause network congestion, slow replica synchronization, or impact cluster stability during Raft snapshot operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 287, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Processed Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows processed by TiDB's Time-To-Live (TTL) feature, broken down by SQL operation type and result status. Essential for tracking TTL job performance, identifying processing bottlenecks, and ensuring expired data is being cleaned up at the expected rate.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 703, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag duration percentile", "group": "Slow Table", "used_for": "Used for monitoring the 99th percentile replication lag duration of slow tables in TiCDC changefeeds, tracking both resolved timestamp and checkpoint timestamp lag. Critical for identifying tables that are falling behind in data replication and diagnosing CDC performance bottlenecks that could impact downstream data freshness.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573350, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring CPU usage of SST (Sorted String Table) file processing threads in TiKV during Point In Time Restore operations. Helps identify CPU bottlenecks and resource contention when restoring data from backups, enabling optimization of restore performance.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 65, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the write throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify write bandwidth bottlenecks, verify if workloads are approaching EBS volume throughput limits, and diagnose I/O performance issues in cloud storage.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 91, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache feature, broken down by cache type (hit/miss). Helps evaluate plan cache effectiveness and identify opportunities to improve query performance by increasing cache hit rates.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 303, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of expired rows successfully deleted by TTL (Time To Live) jobs per hour in TiDB. Helps track TTL cleanup efficiency and identify potential issues when expired data isn't being purged as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573704, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiKV, including sending, applying, and generating snapshots during Raft replication. Critical for diagnosing slow region recovery, replica synchronization delays, and identifying performance bottlenecks when TiKV nodes handle snapshot transfers for data rebalancing or new replica creation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3690, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends committing Raft logs to disk. Critical for diagnosing write path performance issues and identifying disk I/O bottlenecks that could impact transaction commit latency and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 258, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Miss OPS", "group": "Executor", "used_for": "Used for monitoring the rate of plan cache misses per second in TiDB, categorized by miss type. Essential for diagnosing query performance issues caused by inefficient plan cache utilization, which forces TiDB to regenerate execution plans and increases query latency.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 43, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pairs accessed per snapshot operation in TiKV. Critical for identifying extreme cases where transactions read unusually large numbers of keys, which can indicate inefficient queries or potential performance bottlenecks in snapshot isolation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13376, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations across TiKV instances, including overall append duration, KV database writes, and Raft database writes. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow Raft log persistence that could impact cluster consensus and data replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_kvdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_send_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_callback_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 271, "dashboard": "Prometheus-TiCDC-Clinic", "title": "On disk data size", "group": "DB", "used_for": "Used for monitoring the total size of data stored on disk by TiCDC's sorter component per instance. Essential for tracking disk space usage during change data capture operations and identifying potential disk capacity issues or abnormal data accumulation.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4201, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot transport speed", "group": "Snapshot", "used_for": "Used for monitoring the rate of snapshot data transfer and generation in TiKV, measured in bytes per second. Essential for diagnosing snapshot replication performance issues, identifying bandwidth bottlenecks during region migrations, and understanding the speed of data synchronization between TiKV nodes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "rate(tikv_snapshot_limit_transport_bytes{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tikv_snapshot_limit_generate_bytes{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1422, "dashboard": "PD-Cluster-Clinic", "title": "MVCC DB total size", "group": "etcd", "used_for": "Used for monitoring the total size and actual usage of PD's embedded etcd MVCC database. Critical for tracking etcd storage growth and identifying potential space issues that could impact cluster metadata management and PD stability.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}", "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3688, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations on each TiKV instance. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or Raft replication delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying slow scan operations that may impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 284, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Scan Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution (p50, p80, p90, p99) of TTL scan SELECT queries in TiDB. Essential for diagnosing performance issues with Time-To-Live background scanning operations and identifying when expired data cleanup is impacting query response times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))\n", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573352, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of import RPC operations during Point In Time Restore (PITR) operations in TiKV. Essential for tracking restore progress and diagnosing performance bottlenecks during data recovery, particularly when apply operations show abnormal rates or when comparing different RPC request types.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[$__rate_interval])) by (instance, request)", "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (request)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB Cluster Pods", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for displaying all TiDB cluster pods running in the specified Kubernetes namespace. Helps operators quickly verify pod deployment status and identify which pods are active in the cluster for troubleshooting infrastructure or resource allocation issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "namespace" ], "promql": [ "kube_pod_info{namespace=\"$namespace\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 31, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends applying Raft logs to the storage engine. Critical for diagnosing write path performance issues and identifying bottlenecks in the Raft log application process that could impact write throughput and latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 39, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "", "used_for": "Used for monitoring the latency of saving SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance bottlenecks and identifying slow file write operations that could impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 90, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits at the cluster level. Critical for identifying storage bottlenecks and determining when EBS volume capacity needs to be increased to prevent performance degradation.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 273, "dashboard": "Prometheus-TiCDC-Clinic", "title": "In-memory data size", "group": "DB", "used_for": "Used for monitoring the amount of data held in memory by TiCDC's sorter component per instance. Critical for diagnosing memory pressure issues and determining if TiCDC needs memory tuning or if data is being efficiently spilled to disk during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573351, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "P99 RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the 99th percentile RPC latency during Point In Time Restore (PITR) operations in TiKV, including apply requests, download queue/execution times, and engine operations. Critical for diagnosing performance bottlenecks and slow restore operations when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[2m])) by (le, request))", "histogram_quantile(0.99, avg(rate(tikv_import_apply_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"queue|exec_download\"}[2m])) by (le, type))", "histogram_quantile(0.99, avg(rate(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5, "dashboard": "PD-Cluster-Clinic", "title": "Handle transactions rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd transaction operations handled by PD instances, broken down by success/failure results. Essential for diagnosing PD's etcd transaction throughput and identifying transaction processing bottlenecks or failures that could impact cluster metadata operations.", "component": "PD", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 285, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution of TTL (Time To Live) delete queries in TiDB across multiple percentiles (p50, p80, p90, p99). Essential for diagnosing performance issues with automatic data expiration operations and identifying when TTL delete queries are experiencing slowdowns that could impact background cleanup efficiency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 249, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Table Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of read operations served directly from TiDB's table cache feature. Helps assess table cache effectiveness and identify when cached small tables are being accessed, which should show significantly faster read performance compared to regular table reads.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_read_from_tablecache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573032, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cache Events", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of cache events during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose cache behavior and performance during data import and restore processes, identifying potential bottlenecks in cache management.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_apply_cache_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot Actions", "group": "Snapshot", "used_for": "Used for monitoring Raft snapshot operations in TiKV, including snapshot generation, application, and region cleanup activities. Essential for diagnosing region replication issues, identifying snapshot transfer bottlenecks, and troubleshooting scenarios involving region splits, merges, or peer recovery.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_snapshot_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, status)", "sum(delta(tikv_raftstore_clean_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "PD-Cluster-Clinic", "title": "99% Handle transactions duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in PD. Critical for diagnosing slow transaction processing that can impact cluster metadata operations and overall cluster coordination performance.", "component": "PD", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 276, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) scan workers in TiDB. Helps identify which phase (scanning, deleting, or waiting) is consuming the most time, enabling diagnosis of TTL job performance bottlenecks and optimization opportunities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan_worker\"}[2m])) by (phase)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 269, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Memory Usage", "group": "Executor", "used_for": "Used for monitoring the memory consumption of TiDB's plan cache on each instance. Essential for diagnosing memory pressure issues and determining if plan cache memory usage is contributing to OOM problems or needs tuning.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "", "used_for": "Used for monitoring the latency distribution when creating connections to external storage systems (like S3, GCS) during backup and import operations in TiKV. Critical for diagnosing slow backup/restore initialization issues and external storage connectivity problems.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 271, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Plan Num", "group": "Executor", "used_for": "Used for monitoring the total number of cached execution plans in TiDB's plan cache per instance. Helps diagnose plan cache effectiveness and memory usage, particularly when investigating query performance issues or determining if the plan cache size needs adjustment.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_plan_num_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 272, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Level files", "group": "DB", "used_for": "Used for monitoring the distribution of SSTable files across different levels in TiCDC's embedded database (likely LevelDB/RocksDB). Helps diagnose compaction issues, storage health, and potential performance degradation when files accumulate in higher levels or show abnormal distribution patterns.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 6, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Duration", "group": "", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, showing p95, p99, and average durations. Critical for diagnosing backup performance issues and identifying slow backup ranges that may impact backup completion time or cluster performance during backup operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 32, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs to the state machine in TiKV instances. Critical for diagnosing write path performance issues and identifying instances experiencing slow log application that could impact data replication and write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk wait latency", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait latency per I/O operation on EBS volumes in dedicated dataplane nodes, tracking both read and write operations separately. Essential for diagnosing storage performance bottlenecks and identifying when disk I/O latency impacts cluster performance, particularly useful when investigating slow queries or high storage layer delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)", "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 72, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the average disk read wait latency on TiKV nodes at the EBS volume level. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk reads in cloud-deployed TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 59, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiKV, broken down by worker name. Helps identify worker thread bottlenecks, uneven task distribution, and diagnose performance issues related to specific background operations like apply, scheduler, or raftstore workers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573348, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Overall RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the overall RPC duration distribution for Point-in-Time Restore (PITR) apply operations across TiKV instances. Essential for diagnosing performance bottlenecks and latency issues during backup restoration processes, helping identify slow apply requests that may impact recovery time objectives.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 76, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait time for disk write operations on TiKV nodes' EBS volumes. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk writes at the infrastructure level.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 281, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Job Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) jobs across different statuses in TiDB. Helps identify job execution patterns, detect stuck or failed TTL jobs, and diagnose issues with automatic data cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1876, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiKV's FuturePool thread pools. Helps identify thread pool utilization patterns and diagnose potential bottlenecks when certain pools are overloaded or underutilized.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573558, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read File into Memory Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the time taken to read and download backup files into memory during Point-in-Time Restore operations in TiKV. Critical for diagnosing slow restore performance and identifying bottlenecks in the file download phase of the restore process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_apply_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"exec_download\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1395, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks queued in TiKV's various worker threads (such as scheduler, apply, and raftstore workers). Essential for identifying thread pool saturation and task processing bottlenecks that can lead to request timeouts and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) delete workers in TiDB. Helps identify performance bottlenecks in the TTL deletion process by showing which phase (e.g., scanning, deleting, waiting) consumes the most time.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"delete_worker\"}[2m])) by (phase)\n" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 274, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "DB", "used_for": "Used for monitoring the write latency distribution of TiCDC's sorter database operations. Essential for diagnosing performance bottlenecks in TiCDC's data sorting and buffering layer, particularly when replication lag increases or changefeed processing slows down.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13382, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store io task reschedule", "group": "Raft IO", "used_for": "Used for monitoring the frequency of Raft IO task rescheduling in TiKV store threads, tracking both region-level reschedules and pending task reschedules. Essential for diagnosing IO thread contention and identifying when the store thread is overloaded, causing tasks to be rescheduled rather than executed immediately.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", "sum(tikv_raftstore_io_reschedule_pending_tasks_total{instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13380, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Write task block duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile blocking duration of Raft write tasks per TiKV instance. Critical for diagnosing write performance bottlenecks caused by Raft message queue congestion or I/O contention in the storage layer.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 288, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write speed", "group": "DB", "used_for": "Used for monitoring the write throughput of TiCDC's sorter database, showing both total bytes written and write speed per instance. Essential for diagnosing TiCDC performance bottlenecks related to disk I/O when processing change data capture workloads.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 7, "dashboard": "PD-Cluster-Clinic", "title": "99% WAL fsync duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile fsync latency of etcd's Write-Ahead Log (WAL) operations in PD. Critical for diagnosing PD performance issues caused by slow disk I/O, as high WAL fsync duration directly impacts cluster metadata operations and can lead to leader election delays or cluster instability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 311, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Counter", "group": "Executor", "used_for": "Used for monitoring the rate of MPP (Massively Parallel Processing) coordinator operations in TiDB, broken down by operation type. Essential for diagnosing MPP query execution issues and understanding the distribution of coordinator activities when troubleshooting analytical query performance problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_mpp_coordinator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 294, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Task Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) tasks across different statuses in TiDB. Helps track task execution progress and identify stuck or failed TTL cleanup operations that may impact data lifecycle management.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 286, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CPU usage", "group": "DB", "used_for": "Used for monitoring CPU usage of TiCDC sorter workers, which handle data sorting during change data capture replication. Essential for identifying CPU bottlenecks in the sorter component that may impact replication performance and throughput.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[2m])) by (name, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1877, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiKV's FuturePool thread pools. Helps identify thread pool saturation and task queuing issues that can lead to processing delays and performance degradation in TiKV operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average read latency of EBS volumes serving TiFlash nodes in dedicated dataplane deployments. Critical for identifying storage I/O bottlenecks that impact TiFlash's analytical query performance and diagnosing slow disk read operations at the infrastructure level.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763573229, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Queuing Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time distribution that Point In Time Restore (PITR) import requests spend waiting in the queue before processing in TiKV. Helps identify bottlenecks and resource contention during backup restoration operations, particularly when restore tasks are queued due to limited processing capacity.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queuing\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 34, "dashboard": "PD-Cluster-Clinic", "title": "99% Peer round trip time seconds", "group": "etcd", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile network round-trip time between etcd peers in the PD cluster. Critical for diagnosing network latency issues, etcd consensus performance problems, and identifying slow peer communications that could impact PD's availability and cluster metadata operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, To, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 300, "dashboard": "TiDB-Cluster-Clinic", "title": "Table Count By TTL Schedule Delay", "group": "TTL", "used_for": "Used for monitoring the number of tables grouped by their TTL (Time To Live) scheduling delay in TiDB. Helps identify tables experiencing delayed TTL job execution, which is critical for diagnosing data cleanup performance issues and ensuring timely expiration of time-sensitive data.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_server_ttl_watermark_delay{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", type=\"schedule\"}) by (type, name)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 312, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Latency", "group": "Executor", "used_for": "Used for monitoring the latency of MPP (Massively Parallel Processing) coordinator operations in TiDB at different percentiles (p50, p75, p95). Essential for diagnosing performance issues in distributed analytical queries that use TiFlash, helping identify coordination overhead and bottlenecks in MPP query execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 75, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.75, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 151, "dashboard": "PD-Cluster-Clinic", "title": "99% backend commit duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd backend commit operations in PD. Critical for diagnosing PD performance issues caused by slow disk writes, as high commit latency directly impacts cluster metadata operations and can indicate storage bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze coprocessor requests in TiKV during backup and import operations. Critical for diagnosing performance bottlenecks when TiDB performs table checksums or statistics collection, helping identify slow data verification processes that may impact backup/restore operations.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 13, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes or indicate disk saturation issues affecting cluster performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 277, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write bytes", "group": "DB", "used_for": "Used for monitoring the rate of bytes written to TiCDC's sorter database storage. Helps diagnose I/O bottlenecks and storage write performance issues in the change data capture pipeline, particularly when replication lag occurs.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 44, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk wal fsync rate", "group": "etcd", "used_for": "Used for monitoring the rate of WAL (Write-Ahead Log) fsync operations in PD's etcd storage over 2-minute intervals. Essential for identifying write throughput patterns and diagnosing potential disk I/O bottlenecks that could impact PD's ability to persist cluster metadata changes.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763573349, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Throughput", "group": "Point In Time Restore", "used_for": "Used for monitoring the throughput of apply requests during Point In Time Restore (PITR) operations in TiKV, measured in bytes per second. Essential for tracking restore performance and identifying bottlenecks when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_apply_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "", "used_for": "Used for monitoring the p99 latency of external storage connection initialization during backup and restore operations. Critical for diagnosing slow backup/restore startup issues caused by delays in establishing connections to external storage systems like S3 or GCS.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 152, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk backend commit rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd backend commit operations in PD over 2-minute intervals. Essential for tracking etcd write throughput and identifying potential bottlenecks in PD's metadata storage layer that could impact cluster coordination performance.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 108, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile number of Raft proposals processed per ready batch in TiKV. Helps diagnose Raft apply efficiency and identify instances with abnormal proposal batching, which can indicate write throughput bottlenecks or consensus layer performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 73, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Bandwidth", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring disk read and write bandwidth across EBS volumes in the dedicated dataplane cluster. Essential for identifying disk I/O bottlenecks, detecting abnormal storage traffic patterns, and ensuring EBS volumes are not approaching their throughput limits during heavy workloads.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])", "rate(node_disk_written_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763573344, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Downloaded File Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of file sizes downloaded during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose restore performance issues by tracking download volume patterns and identifying unusually large file transfers that may impact recovery time.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_download_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 92, "dashboard": "PD-Cluster-Clinic", "title": "Raft term", "group": "etcd", "used_for": "Used for monitoring the Raft term number in PD's embedded etcd cluster. Helps identify leader elections and cluster stability issues—frequent term changes indicate leadership instability or network partitions affecting consensus.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"term\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 304, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert/Delete Rows By Day", "group": "TTL", "used_for": "Used for monitoring the daily volume of TTL (Time To Live) operations, tracking both rows inserted into TTL tables and expired rows successfully deleted. Essential for understanding TTL workload patterns, verifying automatic data cleanup is functioning correctly, and identifying potential issues with expired data accumulation or deletion performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 7, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft Propose", "used_for": "Used for monitoring the rate of different types of Raft proposals (local reads, normal writes, and read index requests) in TiKV. Essential for understanding read/write workload distribution and diagnosing Raft consensus performance issues or imbalanced request patterns across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Duration", "group": "Distsql", "used_for": "Used for monitoring the latency distribution of distributed SQL (DistSQL) query execution in TiDB at various percentiles (p50, p90, p99, p999). Essential for diagnosing performance issues when TiDB pushes query operations down to TiKV, helping identify slow distributed query processing and coprocessor request bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql QPS", "group": "Distsql", "used_for": "Used for monitoring the distributed SQL query throughput in TiDB, broken down by coprocessor request type (table scan, index scan, etc.). Essential for identifying query load patterns and diagnosing performance issues related to distributed query execution between TiDB and TiKV.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (copr_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3062, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the distribution of coprocessor request latency in TiKV across all instances. Essential for diagnosing slow coprocessor operations that handle table scans, aggregations, and other pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import CPU Utilization", "group": "Import", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import operations. Essential for identifying CPU bottlenecks and resource contention when importing large datasets, helping diagnose slow import performance and optimize thread pool configurations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 313, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Event Count Per Minute", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) events occurring per minute in TiDB, broken down by event type. Essential for tracking TTL job activity, identifying abnormal deletion patterns, and diagnosing issues with automatic data expiration and cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"hook.tidb.ttl\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"runtime\\\\.ttl.*\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573233, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Batch Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of batch sizes during Point-in-Time Restore (PITR) apply operations in TiKV. Helps diagnose restore performance by showing how data is being applied in batches, which is critical for identifying inefficient batch processing or bottlenecks during recovery operations.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_apply_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write OPS", "group": "DB", "used_for": "Used for monitoring the write operations per second to TiCDC's sorter database across instances. Essential for tracking TiCDC's data processing throughput and identifying instances with abnormal write activity or performance bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 16, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the 99th percentile and maximum latency of coprocessor requests in TiKV, broken down by request type. Critical for diagnosing slow analytical queries and table scan performance issues, as coprocessor handles pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 275, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write delay", "group": "DB", "used_for": "Used for monitoring the write delay in TiCDC's downstream database operations. Essential for diagnosing replication lag issues and identifying bottlenecks when TiCDC writes change data to the target database, helping troubleshoot slow data synchronization problems.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 56, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Thread Count", "group": "Import", "used_for": "Used for monitoring the number of active SST import threads per TiKV instance during data import operations. Helps diagnose import performance issues and verify that import parallelism is properly configured across the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 119, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft read proposals (local reads and read index operations) per TiKV instance. Helps identify read traffic distribution across the cluster and diagnose imbalanced read workloads or instances experiencing abnormally high read proposal rates.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573230, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blocked by Concurrency Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time that Point In Time Restore (PITR) operations spend waiting for concurrency permits in TiKV's import engine. Helps diagnose restore performance bottlenecks caused by concurrency limits, indicating when parallel restore operations are being throttled due to resource constraints.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"get_permit\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Partial QPS", "group": "Distsql", "used_for": "Used for monitoring the rate of partial distributed SQL scan operations per second in TiDB. Helps identify when queries are performing partial scans (scanning subset of data), which is useful for diagnosing query execution patterns and potential performance issues related to data distribution or query optimization.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 85, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP In Use", "group": "", "used_for": "Used for monitoring the number of TCP sockets currently in use on dedicated dataplane nodes. Helps identify socket exhaustion issues, connection leaks, or abnormal connection patterns that could impact network communication and service availability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "node_sockstat_TCP_inuse{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 78, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average disk write wait latency for TiFlash nodes at the cluster level. Helps identify storage I/O bottlenecks and disk performance degradation that could impact TiFlash's columnar storage operations and analytical query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(node)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 12842, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Pipeline Duration", "group": "Duration", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency of TiKV's write pipeline stages, including Raft log append duration, request wait time, and apply wait time. Critical for diagnosing write performance bottlenecks and identifying which stage of the write path is causing slowdowns in data persistence operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_msg_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 93, "dashboard": "PD-Cluster-Clinic", "title": "Raft committed index", "group": "etcd", "used_for": "Used for monitoring the Raft committed index in PD's embedded etcd cluster. Essential for tracking etcd consensus progress and diagnosing replication lag or split-brain issues when the committed index stops advancing or diverges across PD nodes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"committedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 27, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Errors", "group": "Import", "used_for": "Used for monitoring the rate of import errors occurring in TiKV during data import operations. Essential for quickly identifying and diagnosing import failures, data ingestion issues, or configuration problems that may interrupt bulk data loading processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 74, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Requests", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor requests per second in TiKV, broken down by request type. Essential for tracking coprocessor workload patterns and identifying which types of requests (table scan, index scan, etc.) are dominating cluster activity.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573118, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Speed", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of Point-in-Time Restore (PITR) apply requests being initiated in TiKV. Helps track the speed of log replay during restore operations and diagnose slow recovery performance or bottlenecks in the restore process.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_applier_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"begin_req\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 120, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiKV instance. Helps identify unbalanced write workloads across TiKV nodes and diagnose potential Raft layer bottlenecks or hotspots when certain instances show significantly higher proposal rates than others.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 38, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in TiKV's default column family during backup operations. Helps diagnose backup performance issues and identify abnormally large SST files that could impact backup efficiency and storage requirements.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 94, "dashboard": "PD-Cluster-Clinic", "title": "Raft applied index", "group": "etcd", "used_for": "Used for monitoring the Raft applied index in PD's embedded etcd cluster, which tracks the last log entry applied to the state machine. Essential for diagnosing etcd replication lag, consensus issues, and ensuring all PD nodes have consistently applied the same set of configuration changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"appliedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 289, "dashboard": "TiDB-Cluster-Clinic", "title": "GOGC", "group": "Resource Manager", "used_for": "Used for monitoring the GOGC (Go Garbage Collection) parameter value in TiDB instances. Helps diagnose memory management behavior and tune garbage collection aggressiveness when investigating memory pressure or GC-related performance issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per distributed SQL query at different percentiles (p50, p90, max). Essential for identifying inefficient queries with excessive key scans, which can indicate missing indexes, full table scans, or poorly optimized query patterns causing performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 49, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import RPC Duration", "group": "Import", "used_for": "Used for monitoring the 99th percentile latency of import RPC requests in TiKV during data import operations. Critical for diagnosing performance bottlenecks and slow import operations when bulk loading data into the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 87, "dashboard": "Dedicated Dataplane Node Info", "title": "Segments retransmitted", "group": "", "used_for": "Used for monitoring the rate of TCP segment retransmissions on dedicated dataplane nodes. Helps diagnose network reliability issues, packet loss, or congestion problems that may impact cluster communication and overall performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "rate(node_netstat_Tcp_RetransSegs{instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 89, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP Connections", "group": "", "used_for": "Used for monitoring the current number of established TCP connections on dedicated dataplane nodes. Helps identify connection leaks, unusual connection patterns, or capacity issues when connection counts spike unexpectedly.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance" ], "promql": [ "node_netstat_Tcp_CurrEstab{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 282, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - Next", "group": "DB", "used_for": "Used for monitoring the latency distribution of iterator \"next\" operations when reading from TiCDC's sorter database. Critical for diagnosing performance bottlenecks in TiCDC's event sorting and processing pipeline, especially when replication lag increases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 41, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft proposals spend waiting in the queue before being processed in TiKV. Critical for diagnosing write latency issues caused by proposal queue congestion or slow Raft processing, helping identify when the Raft layer becomes a bottleneck for write operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3128, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Request Errors", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV, broken down by error reason. Essential for diagnosing query execution failures and identifying specific error patterns (such as lock conflicts, region errors, or timeout issues) that may be impacting application performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 293, "dashboard": "TiDB-Cluster-Clinic", "title": "EMA CPU Usage", "group": "Resource Manager", "used_for": "Used for monitoring the Exponential Moving Average (EMA) of CPU usage by TiDB's Resource Manager. Helps track smoothed CPU consumption trends to diagnose resource allocation issues and identify whether resource groups are consuming CPU as expected over time.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573346, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cached File in Memory", "group": "Point In Time Restore", "used_for": "Used for monitoring the amount of cached file data held in memory during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose memory pressure issues and optimize restore performance by tracking how much imported data is buffered before being applied to storage.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_import_apply_cached_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573119, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Engine Requests Unfinished", "group": "Point In Time Restore", "used_for": "Used for monitoring the accumulation of unfinished engine requests during Point In Time Restore (PITR) operations in TiKV. Helps identify bottlenecks or stalled restore processes by tracking requests that have been initiated but not yet completed, which is critical for diagnosing PITR performance issues or hangs.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "increase(tikv_import_applier_event{instance=~\"$instance\", type!=\"begin_req\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of keys scanned in partial results during distributed SQL execution at p80 and p90 percentiles. Helps diagnose query efficiency issues by identifying queries that scan excessive keys, which may indicate missing indexes or inefficient query plans causing performance degradation.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 281, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - Release", "group": "DB", "used_for": "Used for monitoring the latency of releasing read iterators in TiCDC's sorter database component. Helps diagnose performance bottlenecks in the sorter's resource cleanup operations, which can impact overall CDC replication efficiency when release operations become slow.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 19, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "Import", "used_for": "Used for monitoring disk I/O utilization percentage during import operations. Helps identify storage bottlenecks and determine if disk I/O is saturated, which can significantly impact import performance and overall cluster throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 46, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "Import", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets, helping identify whether slow imports are caused by write or download operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 42, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Propose wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed on each TiKV instance. Critical for diagnosing Raft layer bottlenecks and identifying instances experiencing proposal queuing delays that impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of key scanning operations performed by TiKV coprocessor requests. Helps diagnose coprocessor performance issues by tracking how many keys are being scanned per second, which is essential for identifying inefficient queries or table scans causing high I/O load.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12970, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cop Read Duration", "group": "Duration", "used_for": "**Used For:**\n\nUsed for monitoring the end-to-end latency of coprocessor read requests in TiKV at p95 and p99 percentiles, including snapshot acquisition, queue wait time, and request processing duration. Critical for diagnosing slow analytical queries and identifying whether performance bottlenecks occur during storage access, request queuing, or coprocessor execution phases.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 278, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - First", "group": "DB", "used_for": "Used for monitoring the latency of the first read operation in TiCDC's sorter database iterator. Helps diagnose performance issues in TiCDC's data sorting mechanism, particularly when investigating delays in change data capture processing or replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3129, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the number of keys scanned per coprocessor request at the 99th percentile and maximum levels in TiKV. Essential for diagnosing inefficient queries with excessive key scans, identifying full table scans, and detecting coprocessor performance bottlenecks caused by scanning too many keys.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store size", "group": "Cluster", "used_for": "Used for monitoring storage space utilization across TiKV instances, breaking down total used space into KV data and Raft log sizes. Essential for capacity planning, identifying storage imbalances between nodes, and detecting abnormal storage growth that could lead to disk space exhaustion.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"used\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"kv_size\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"raft_size\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"import_size\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 142, "dashboard": "PD-Cluster-Clinic", "title": "Handle requests duration", "group": "TiDB", "used_for": "Used for monitoring the 98th percentile latency of TiDB's requests to PD (Placement Driver), broken down by request type. Critical for diagnosing communication delays between TiDB and PD, which can indicate PD overload, network issues, or cluster coordination problems affecting query performance.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 98 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763573231, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time distribution of apply operations during Point-in-Time Restore (PITR) in TiKV. Helps diagnose performance bottlenecks and slow apply phases when restoring data to a specific point in time.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1706, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Available size", "group": "Cluster", "used_for": "Used for monitoring the available disk space on each TiKV instance. Critical for preventing storage exhaustion issues and planning capacity expansion before nodes run out of space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"available\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "TiDB-Cluster-Clinic", "title": "Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of partial results generated during distributed SQL execution in TiDB, showing p50, p90, and maximum values. Helps diagnose query execution efficiency and identify queries that require excessive data shuffling or produce fragmented results across multiple TiKV regions.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 280, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - First", "group": "DB", "used_for": "Used for monitoring the rate of first-read operations per second in TiCDC's sorter database iterator. Helps identify throughput patterns and potential bottlenecks when TiCDC initializes data reads during change data capture processing.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 29, "dashboard": "PD-Cluster-Clinic", "title": "PD server TSO handle time", "group": "TiDB", "used_for": "**Used For:**\n\nUsed for monitoring the latency of PD server handling TSO (Timestamp Oracle) requests at various percentiles (p90, p99, p999, p99999). Critical for diagnosing transaction performance issues, as high TSO handle times directly impact TiDB's ability to assign timestamps for distributed transactions and can indicate PD server bottlenecks or network problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [ 90, 99, 999, 99999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 398, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with source database changes in real-time.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13524, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the wait time before store write tasks are executed in TiKV's Raftstore. Critical for diagnosing write path bottlenecks and identifying when the store thread is overloaded, causing queuing delays that impact overall write latency.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 310, "dashboard": "TiDB-Cluster-Clinic", "title": "Closest Replica Hit Count Per Second", "group": "Follower Read", "used_for": "Used for monitoring the rate of queries that successfully hit the closest replica when follower read is enabled in TiDB. Helps verify follower read effectiveness and diagnose whether queries are properly utilizing nearby replicas to reduce cross-AZ network latency.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_closest_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 175, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Cache", "group": "Distsql", "used_for": "Used for monitoring the coprocessor cache hit/miss rates in TiDB's distributed SQL execution. Helps identify whether coprocessor requests are being efficiently cached, which is crucial for optimizing query performance and reducing unnecessary data transfers from TiKV.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 283, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - Next", "group": "DB", "used_for": "Used for monitoring the rate of iterator \"next\" operations in TiCDC's sorter database per instance. Helps identify sorter read throughput and detect performance bottlenecks when TiCDC processes change data during replication.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 59, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Wait Duration", "group": "Import", "used_for": "Used for monitoring the wait time in the import queue during data import operations in TiKV. Helps identify import bottlenecks and queue congestion when bulk loading data into the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 1707, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Capacity size", "group": "Cluster", "used_for": "Used for monitoring the total storage capacity configured for each TiKV instance. Essential for capacity planning and identifying instances approaching their storage limits before they run out of disk space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"capacity\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2118, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total RocksDB Perf Statistics", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of internal delete operations skipped by RocksDB during coprocessor requests in TiKV. Helps diagnose storage engine efficiency and identify potential issues with delete operations or MVCC garbage collection that could impact coprocessor query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "PD-Cluster-Clinic", "title": "Handle requests count", "group": "TiDB", "used_for": "Used for monitoring the rate of TSO (Timestamp Oracle) requests handled by PD server. Helps identify TSO request patterns, detect abnormal spikes in timestamp allocation requests, and diagnose potential bottlenecks in distributed transaction coordination.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 324, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size Per Instance and TiKV", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor response data size from each TiKV store to TiDB instances during follower read operations. Helps identify network bandwidth usage patterns and diagnose potential data transfer bottlenecks between TiDB and TiKV nodes.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Seconds 999", "group": "Distsql", "used_for": "Used for monitoring the 99.9th percentile latency of coprocessor requests sent from TiDB to TiKV. Critical for identifying tail latency issues in distributed query execution, particularly when diagnosing slow analytical queries or table scans that push computation down to TiKV.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{store!=\"0\", scope=\"false\", type=\"Cop\", k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 468, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring downstream systems receive near real-time updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573449, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Store Memory Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring memory consumption of TiKV's Raft Store component, which handles consensus protocol operations and log replication. Critical for diagnosing memory pressure issues during Point In Time Restore operations and identifying instances with abnormal Raft-related memory usage.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 287, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - Release", "group": "DB", "used_for": "Used for monitoring the rate of read operations during sorter database iterator release in TiCDC. Helps identify potential resource cleanup issues or bottlenecks when releasing database iterators during data replication.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 551, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Response Size", "group": "Coprocessor Overview", "used_for": "Used for monitoring the total rate of data returned by TiKV's coprocessor to TiDB across the cluster. Helps identify heavy scan operations, large result sets, and potential network bandwidth bottlenecks when response sizes are unexpectedly high.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13522, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft store write tasks are executed on each TiKV instance. Critical for diagnosing write bottlenecks and identifying when the Raft store thread pool is overloaded or experiencing queueing delays.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2535, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft log entries spend waiting in the apply queue before being applied to the state machine in TiKV. Critical for diagnosing apply thread bottlenecks and identifying when the Raft apply process is falling behind, which can cause replication lag and increased write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 285, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Compact duration", "group": "DB", "used_for": "Used for monitoring the compaction duration of TiCDC's sorter database, which manages event ordering during data replication. Essential for diagnosing TiCDC performance degradation caused by slow database compaction operations that can impact replication latency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1447, "dashboard": "PD-Cluster-Clinic", "title": "Tso Request batch size", "group": "TiDB", "used_for": "Used for monitoring the batch size of TSO (Timestamp Oracle) requests from TiDB to PD, showing p90, p99, and average values. Helps diagnose TSO request batching efficiency and identify potential timestamp allocation bottlenecks that could impact transaction performance.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [ 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "sum(delta(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(delta(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1708, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of individual TiKV instances in the cluster. Essential for identifying CPU bottlenecks, unbalanced load distribution across nodes, and determining if TiKV instances are CPU-saturated during performance degradation.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\".*tikv\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 469, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "", "used_for": "Used for monitoring the rate of event feed errors per minute in TiCDC's KV client, categorized by error type. Critical for identifying replication issues, connection problems, or data synchronization failures that could impact downstream data consistency.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 52, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Read SST Duration", "group": "Import", "used_for": "Used for monitoring the duration of reading SST (Sorted String Table) files during data import operations in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the SST file reading phase of backup restoration or data migration tasks.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 2536, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied to the state machine in TiKV. Critical for diagnosing apply thread bottlenecks and identifying instances experiencing delays in committing Raft proposals to storage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 325, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size", "group": "Follower Read", "used_for": "Used for monitoring the size of coprocessor responses returned from TiKV to TiDB, showing average, median (p50), p80, and p99 percentiles. Essential for diagnosing network bandwidth issues, identifying queries that transfer excessive data, and understanding the I/O characteristics of distributed query execution in follower read scenarios.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_distsql_copr_resp_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.5, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 24763573286, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Quota", "group": "Coprocessor Overview", "used_for": "Used for monitoring the memory quota allocated to TiKV's coprocessor component across instances. Helps identify memory allocation patterns and potential memory pressure issues when coprocessor requests consume excessive resources, which can lead to OOM errors or query failures.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 50, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "Import", "used_for": "Used for monitoring the duration of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks in the import pipeline, particularly when ingesting large volumes of data through backup restoration or bulk import operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 8385, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolved TS Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's Resolved TS worker threads, which maintain transaction timestamp consistency for CDC and other features. Essential for diagnosing performance bottlenecks in change data capture pipelines and identifying instances where Resolved TS processing is consuming excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"resolved_ts.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 466, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD operator/m", "group": "", "used_for": "Used for monitoring the rate of PD scheduling operators created per minute, broken down by operator type. Helps identify scheduling activity patterns and diagnose issues related to region balancing, leader transfers, and other cluster rebalancing operations in TiDB.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1709, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring the resident memory usage of each TiKV instance in the cluster. Essential for detecting memory leaks, identifying instances approaching memory limits, and diagnosing out-of-memory issues that could cause TiKV crashes or performance degradation.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tikv\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "Import", "used_for": "Used for monitoring the latency distribution of ingest RPC requests during data import operations in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets, particularly when using tools like TiDB Lightning or BR (Backup & Restore).", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 1710, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring the percentage of time that disk I/O operations are in progress on TiKV nodes. Critical for identifying disk bottlenecks and I/O saturation issues that can severely impact TiKV read/write performance and overall cluster stability.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff Duration", "group": "KV Errors", "used_for": "Used for monitoring the duration of KV client backoff operations at different percentiles (p80, p99, p999) when TiDB encounters errors communicating with TiKV. Essential for diagnosing performance degradation caused by region unavailability, lock conflicts, or other transient TiKV errors that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 15, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed count", "group": "KVClient", "used_for": "Used for monitoring the number of active EventFeed gRPC connections between TiCDC and TiKV nodes. Essential for diagnosing change data capture issues, detecting connection failures, and ensuring TiCDC is properly receiving data change events from all TiKV instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 9162, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advance ts Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's advance timestamp worker threads in the Resolved-TS subsystem. Critical for diagnosing performance bottlenecks in CDC (Change Data Capture) and transaction timestamp advancement, especially when replication lag or resolved timestamp progression issues occur.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"advance_ts.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572700, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write handle msg duration", "group": "Raft Propose", "used_for": "Used for monitoring the time spent handling write messages in TiKV's store thread, which processes Raft log writes and applies committed entries. Critical for diagnosing write path bottlenecks and identifying when the store thread becomes a performance bottleneck due to slow disk I/O or high write load.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_handle_msg_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572701, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write trigger size", "group": "Raft Propose", "used_for": "Used for monitoring the distribution of write batch sizes that trigger store-level writes in TiKV's Raft store. Helps diagnose write batching efficiency and identify whether write operations are being triggered at optimal sizes, which impacts I/O performance and write amplification.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_trigger_wb_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat region event QPS", "group": "Heartbeat", "used_for": "Used for monitoring the rate of region heartbeat events received by PD from TiKV stores. Essential for diagnosing region scheduling issues, detecting abnormal region state changes, and identifying stores with excessive region operations that may impact cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 11, "dashboard": "TiDB-Cluster-Clinic", "title": "TiClient Region Error OPS", "group": "KV Errors", "used_for": "**Used For:**\n\nUsed for monitoring the rate of region-level errors encountered by TiDB when accessing TiKV, broken down by error type (including server_is_busy). Critical for diagnosing region availability issues, hotspot problems, and TiKV overload conditions that impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 53, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of KV client backoff operations in TiDB, categorized by backoff type. Essential for diagnosing TiKV communication issues, region unavailability, or lock conflicts that force TiDB to retry operations with exponential backoff.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 479, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiDB query duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile query duration in TiDB, excluding internal queries and simple commands. Critical for identifying tail latency issues and detecting slow query outliers that may impact user experience, particularly useful when diagnosing performance degradation in TiCDC replication scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[2m])) by (le, sql_type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 113, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the P99 latency of coprocessor request handling in TiKV, broken down by request type. Critical for diagnosing slow analytical queries and table scan performance issues, as coprocessor handles pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 316, "dashboard": "TiDB-Cluster-Clinic", "title": "Total encode/deliver/import-kv speed", "group": "Import Into", "used_for": "Used for monitoring the throughput of IMPORT INTO operations across three stages: data encoding (restored), delivery to TiKV (written), and final import completion (imported). Essential for diagnosing bottlenecks in bulk data loading operations and identifying which stage is limiting overall import performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}[2m])) by(task_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 140, "dashboard": "PD-Cluster-Clinic", "title": "99% Region heartbeat handle latency", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling region heartbeat requests from TiKV stores. Critical for diagnosing PD performance bottlenecks and identifying when the cluster's metadata management is experiencing delays that could impact scheduling decisions and cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 9164, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scan lock Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's incremental scan lock workers that support Resolved-TS functionality. Essential for diagnosing performance issues in CDC (Change Data Capture) and transaction consistency features, especially when experiencing high CPU consumption or replication lag.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"inc_scan.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "TiDB-Cluster-Clinic", "title": "Delivered kv count", "group": "Import Into", "used_for": "Used for monitoring the total number of key-value pairs delivered during IMPORT INTO operations in TiDB. Helps track data ingestion progress and diagnose slow import performance by identifying bottlenecks in the KV delivery pipeline.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_block_deliver_kv_pairs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 29, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client receive events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of change events received by TiCDC's KV client from TiKV per second, broken down by instance and event type. Essential for tracking data capture throughput and diagnosing replication lag or event processing bottlenecks in change data capture pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1711, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "MBps", "group": "Cluster", "used_for": "Used for monitoring the read and write throughput (in megabytes per second) of TiKV's RocksDB storage engine across cluster instances. Essential for identifying I/O bottlenecks, unbalanced data distribution, and diagnosing performance issues related to disk bandwidth saturation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[2m])) by (instance)", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 486, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write rows/s", "group": "", "used_for": "Used for monitoring the rate of rows being written to downstream sinks by TiCDC changefeeds. Essential for tracking replication throughput and identifying bottlenecks in data synchronization to downstream databases.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_sink_txn_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed)", "sum (rate(ticdc_sink_ddl_exec_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8387, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum gap between current time and resolved timestamp in TiKV instances. Critical for diagnosing CDC (Change Data Capture) replication lag and identifying instances where transaction resolution is delayed, which can impact downstream data consistency and real-time synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 317, "dashboard": "TiDB-Cluster-Clinic", "title": "Encoded/Delivered/Imported data size", "group": "Import Into", "used_for": "Used for monitoring the data flow during IMPORT INTO operations in TiDB, tracking three key stages: encoded data size (total_restore), data written to storage (written), and data successfully restored (restored). Essential for diagnosing import performance bottlenecks, verifying data processing progress, and identifying discrepancies between encoded, written, and imported data volumes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"total_restore\"}) by(task_id)", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 32, "dashboard": "TiDB-Cluster-Clinic", "title": "Lock Resolve OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of lock resolution operations in TiDB's transaction processing system. Essential for diagnosing transaction conflicts and identifying when the cluster is experiencing high contention, which can indicate write hotspots or long-running transactions blocking others.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 53, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "Import", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets, as high ingestion duration directly impacts overall import speed and cluster write performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 117, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "95% Handle duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the 95th percentile latency of coprocessor request handling across individual TiKV stores. Essential for identifying performance bottlenecks and uneven load distribution when diagnosing slow query issues or hotspot problems in specific TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1460, "dashboard": "PD-Cluster-Clinic", "title": "TiKV side heartbeat statistics", "group": "Heartbeat", "used_for": "**Used For:**\n\nUsed for monitoring heartbeat communication between TiKV nodes and PD, tracking both the rate of heartbeat messages by type and the number of pending heartbeats. Essential for diagnosing cluster coordination issues, detecting network problems between TiKV and PD, and identifying when TiKV nodes are experiencing delays in reporting their status to the placement driver.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1713, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "QPS", "group": "Cluster", "used_for": "Used for monitoring the queries per second (QPS) handled by each TiKV instance, broken down by gRPC message type. Essential for identifying traffic distribution across TiKV nodes, detecting hot instances, and diagnosing throughput bottlenecks or imbalanced workloads in the cluster.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance,type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1975, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft propose speed", "group": "Raft Propose", "used_for": "**Used For:**\n\nUsed for monitoring the average rate of Raft log proposal throughput (bytes per second) across TiKV instances. Helps identify write workload intensity and potential bottlenecks in Raft consensus replication, especially useful when diagnosing slow writes or uneven load distribution across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 111, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the P99 wait duration of coprocessor requests in TiKV before they begin execution. Critical for diagnosing queueing delays and resource contention issues when coprocessor requests are waiting for available threads or resources to process analytical queries.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "KVClient", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's event feed per minute, categorized by error type. Critical for diagnosing data replication issues and identifying whether errors correlate with PD scheduling operations like leader transfers or region/peer changes that may disrupt change data capture.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1977, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft message wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft messages spend waiting in queues before being processed in TiKV. Critical for diagnosing Raft communication bottlenecks and identifying when message processing delays are causing replication lag or consensus slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_msg_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster health problems that may affect region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"ok\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 473, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV min resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiKV's resolved timestamp and the cluster's current TSO (Timestamp Oracle) in TiCDC change data capture scenarios. Critical for identifying data replication delays and diagnosing CDC pipeline bottlenecks that could impact downstream data consistency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573805, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of follower safe-ts", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum time gap between follower replicas' safe timestamps in TiKV's Resolved TS mechanism. Critical for diagnosing replication lag issues and ensuring data consistency across followers, especially when troubleshooting stale read problems or CDC (Change Data Capture) delays.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_follower_safe_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 319, "dashboard": "TiDB-Cluster-Clinic", "title": "Data read/encode/deliver average duration", "group": "Import Into", "used_for": "Used for monitoring the average latency of three key stages in TiDB's IMPORT INTO operation: reading source data, encoding rows, and delivering data blocks. Essential for diagnosing performance bottlenecks during bulk data import operations and identifying which stage is causing slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_import_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_row_read_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_read_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572078, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Resolved TS Region", "group": "Resolved-TS", "used_for": "Used for monitoring the region with the minimum resolved timestamp on each TiKV instance. Critical for diagnosing Resolved TS lag issues and identifying which specific region is blocking CDC/TiFlash data replication progress.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_resolved_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 116, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "95% Wait duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the 95th percentile wait time of coprocessor requests before execution across different TiKV stores. Essential for identifying stores experiencing queueing delays and diagnosing uneven load distribution or resource contention issues that cause requests to wait before processing.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during TiDB data import operations. Helps diagnose import performance bottlenecks by tracking download speeds, essential when troubleshooting slow data restoration or migration tasks.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 1976, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile duration of RocksDB performance context operations during Raft proposal processing in TiKV, including both store and apply threads. Critical for diagnosing write path bottlenecks by identifying which RocksDB internal operations (like WAL writes, memtable operations, or compactions) are causing latency spikes during Raft log replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client dispatch events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of KV events dispatched by TiCDC's KV client, including event send counts and batch resolved event sizes per second. Essential for tracking TiCDC's data capture throughput and identifying bottlenecks in event processing or replication lag issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, namespace,changefeed, type)", "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace, changefeed, table)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 54, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "Import", "used_for": "Used for monitoring the rate of SST (Sorted String Table) file ingestion during data import operations in TiKV. Essential for tracking import throughput and diagnosing slow data ingestion performance during backup restoration or bulk data loading scenarios.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 485, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC sink operations, including both transaction flush duration and DDL execution time when writing to downstream systems. Critical for diagnosing replication lag issues and identifying performance bottlenecks in data synchronization to downstream databases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace,changefeed))", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace, changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 25, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Leader", "group": "Import", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance during import operations. Helps identify leader distribution imbalances and detect sudden leader transfers (drops of 10+ leaders within 2 minutes), which can indicate cluster instability or performance issues during data import.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 1712, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errps", "group": "Cluster", "used_for": "Used for monitoring error rates per second across TiKV instances, including gRPC message failures, PD heartbeat anomalies (noop messages), and critical errors by type. Essential for quickly identifying unhealthy TiKV nodes and diagnosing cluster stability issues or communication failures between components.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)", "sum(delta(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"noop\"}[2m])) by (instance) < 1", "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 320, "dashboard": "TiDB-Cluster-Clinic", "title": "Import/Normal mode", "group": "Import Into", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration during IMPORT INTO operations. Helps verify if TiKV switches between import mode (higher limit for faster bulk loading) and normal mode (standard limit for regular operations), which is critical for diagnosing import performance issues and mode transition problems.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 401, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scanning activity patterns and diagnose potential bottlenecks in TiCDC's data replication process when capturing incremental changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 188, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client gRPC stream count", "group": "KVClient", "used_for": "Used for monitoring the number of active gRPC streams between TiCDC's KV client and each TiKV store. Essential for diagnosing connection issues, detecting stream leaks, and understanding TiCDC's data replication connectivity health across the cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 29, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Region", "group": "Import", "used_for": "Used for monitoring the total number of Raft regions on each TiKV instance during import operations. Helps identify region distribution imbalances and potential hotspots that could impact import performance and cluster stability.", "component": "", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 64, "dashboard": "PD-Cluster-Clinic", "title": "Region schedule push", "group": "Heartbeat", "used_for": "Used for monitoring the rate at which PD pushes region scheduling commands to TiKV stores via heartbeat messages. Essential for diagnosing scheduling delays and identifying stores that may be slow to receive or process region scheduling operations, which can impact cluster rebalancing and data migration performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1715, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance and detecting sudden leader drops. Critical for identifying leader balance issues, region scheduling problems, or potential TiKV node failures when leaders decrease rapidly (>10 regions in 2 minutes).", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3195, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Requests", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) coprocessor requests processed by TiKV instances, broken down by vectorization type. Essential for tracking coprocessor workload intensity and identifying whether queries are using vectorized or non-vectorized execution, which helps diagnose query performance and resource utilization issues.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_dag_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (vec_type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573804, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Safe TS Follower Region", "group": "Resolved-TS", "used_for": "Used for monitoring which TiKV regions have the minimum safe timestamp among follower replicas in the Resolved TS mechanism. Critical for diagnosing stale read issues and identifying regions with lagging replication that may block CDC or backup progress.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_follower_safe_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report error", "group": "Heartbeat", "used_for": "Used for monitoring failed region heartbeat reports from TiKV stores to PD. Critical for diagnosing cluster health issues, network problems, or store failures that prevent PD from receiving timely region status updates, which can impact scheduling decisions and cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"err\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 467, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan region time/m", "group": "", "used_for": "Used for monitoring the total time TiKV spends scanning regions per minute for CDC (Change Data Capture) operations. Helps diagnose CDC performance issues and identify TiKV instances experiencing high region scan overhead during data replication.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1714, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the total number of regions and buckets distributed across each TiKV instance in the cluster. Essential for identifying region distribution imbalances, detecting region split/merge activities, and diagnosing load balancing issues that could impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"buckets\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 223, "dashboard": "TiDB-Cluster-Clinic", "title": "Replica Selector Failure Per Second", "group": "KV Errors", "used_for": "Used for monitoring the rate of replica selector failures per second in TiDB's TiKV client, categorized by failure type. Critical for diagnosing issues with TiDB's ability to select appropriate TiKV replicas for read/write operations, which can indicate network problems, region unavailability, or load balancing issues.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Import", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze requests during data import operations in TiKV. Critical for diagnosing performance bottlenecks during backup restoration or data import processes, especially when checksum verification is taking longer than expected.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 3264, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Executors", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) executor operations in TiKV's coprocessor, broken down by executor type. Helps identify which coprocessor operations (like TableScan, Selection, Aggregation) are most frequently executed and diagnose coprocessor performance bottlenecks or unusual query patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9168, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check leader duration", "group": "Resolved-TS", "used_for": "Used for monitoring the time spent checking Raft leader status during resolved timestamp advancement in TiKV. Critical for diagnosing resolved-ts lag issues caused by slow leader verification, which can impact CDC (Change Data Capture) and backup/restore operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by store", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of KV requests from TiDB to individual TiKV stores. Essential for identifying performance bottlenecks and uneven load distribution across TiKV nodes, helping diagnose which specific stores are experiencing slow request processing.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le, store))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 172, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request OPS", "group": "KV Request", "used_for": "Used for monitoring the operations per second (OPS) of different KV request types sent from TiDB to TiKV, broken down by TiDB instance. Essential for identifying which TiDB nodes are generating the most KV traffic and diagnosing request distribution imbalances or sudden traffic spikes that could impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 5, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft Process", "used_for": "Used for monitoring the rate of Raft ready operations handled by TiKV's Raftstore, broken down by operation type. Essential for diagnosing Raft consensus performance and identifying bottlenecks in the Raft message processing pipeline, particularly when investigating replication delays or I/O issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 407, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV leader change", "group": "", "used_for": "Used for monitoring the rate of Raft leader region changes in TiKV instances over 2-minute intervals. Helps identify cluster instability, frequent leader transfers, or potential network/performance issues causing excessive leader elections.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 131, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report active", "group": "Heartbeat", "used_for": "Used for monitoring the rate of active region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster coordination problems that could impact region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"bind\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 177, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client scanning regions", "group": "KVClient", "used_for": "Used for monitoring the number of regions currently being scanned by TiCDC's KV client during change data capture operations. Helps diagnose CDC performance issues related to region scanning overhead and identify potential bottlenecks when TiCDC is processing large numbers of regions across the cluster.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4106, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring how long each TiKV instance has been running since its last restart. Essential for tracking cluster stability, identifying recently restarted nodes, and correlating service disruptions with instance restarts during troubleshooting.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tikv\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "Import", "used_for": "Used for monitoring disk I/O utilization percentage during import operations. Helps identify storage bottlenecks and determine if disk I/O is saturated, which can significantly impact import performance and overall cluster throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations in TiKV's coprocessor, broken down by scan type (e.g., full table scan, index scan). Essential for identifying inefficient queries that perform excessive table scans and diagnosing coprocessor performance bottlenecks caused by scan-heavy workloads.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573950, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scan backoff duration", "group": "Resolved-TS", "used_for": "Used for monitoring the backoff duration during initial scans in TiKV's Resolved TS mechanism. Helps diagnose delays in establishing resolved timestamps, which can indicate resource contention or slow initial data scanning that impacts CDC (Change Data Capture) and backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571996, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Duration", "group": "Global Sort", "used_for": "**Used For:**\n\nUsed for monitoring the latency of writing data to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage, particularly when investigating slow data import or DDL operations using global sort.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 7235, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Replica read lock checking duration", "group": "Raft Process", "used_for": "Used for monitoring the time spent checking locks during replica read operations in TiKV. Critical for diagnosing performance issues with follower reads and identifying lock contention that may cause replica read latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_replica_read_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by type", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of different KV request types (Get, Scan, Prewrite, Commit, etc.) sent from TiDB to TiKV. Essential for identifying which specific request types are experiencing high latency and diagnosing TiDB-TiKV communication bottlenecks or slow transaction operations.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le,type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 123, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max duration of raft store events", "group": "Raft Process", "used_for": "Used for monitoring the maximum duration of Raft store events in TiKV, including event processing and broadcast operations. Critical for diagnosing Raft consensus performance issues, identifying slow store threads, and detecting bottlenecks in region replication that could impact write latency and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(1.0, sum(rate(tikv_broadcast_normal_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 17, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Event size percentile", "group": "KVClient", "used_for": "Used for monitoring the size distribution of change data capture events at the KV client layer in TiCDC, showing p95 and p999 percentiles. Helps identify abnormally large events that could impact replication performance, memory usage, or indicate issues with specific data change patterns being captured.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10, "dashboard": "PD-Cluster-Clinic", "title": "Storage capacity", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity available across the entire TiDB cluster. Essential for capacity planning and identifying when the cluster is approaching storage limits that could impact write operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763571997, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the write throughput rate (at p50, p95, and p999 percentiles) when TiDB's Global Sort feature writes data to cloud storage during distributed operations like index creation or data import. Critical for diagnosing performance bottlenecks in cloud storage write operations and identifying whether cloud I/O is limiting Global Sort performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1400, "dashboard": "PD-Cluster-Clinic", "title": "99% store heartbeat handle duration", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling store heartbeat messages from TiKV nodes. Critical for diagnosing PD performance bottlenecks and detecting when the cluster controller is slow to process store status updates, which can impact scheduling decisions and cluster health detection.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1402, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful heartbeat messages pushed from PD to TiKV stores. Helps diagnose communication issues between PD and TiKV nodes, particularly when stores fail to receive timely region heartbeat updates.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"ok\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 470, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV admin apply/s", "group": "", "used_for": "Used for monitoring the rate of successful Raft admin commands applied per second in TiKV, broken down by command type (excluding compaction). Essential for tracking cluster administrative operations like region splits, merges, and configuration changes, helping diagnose issues with region management and Raft consensus operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3263, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations in TiKV's coprocessor, broken down by operation type (tag). Essential for diagnosing index scan performance issues and understanding which types of index operations (e.g., processed keys, total keys) are contributing to coprocessor workload.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2741, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiKV instances, categorized by error type. Essential for immediately identifying severe system issues like data corruption, storage failures, or critical internal errors that require urgent attention and may impact cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 219, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded between TiKV stores in TiDB. Helps identify request routing patterns and diagnose issues with store availability, leader transfers, or network problems causing excessive request forwarding between stores.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (from_store, to_store, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 97, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client batch resolved region count", "group": "KVClient", "used_for": "Used for monitoring the distribution of resolved region counts per batch in TiCDC's KV client. Helps diagnose event batching efficiency and identify potential bottlenecks in region resolution processing during change data capture operations.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1403, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report error", "group": "Heartbeat", "used_for": "Used for monitoring errors in PD's heartbeat stream when pushing region information to TiKV stores. Critical for diagnosing communication failures between PD and TiKV that can cause cluster metadata inconsistencies and scheduling problems.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"err\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763572077, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts in region leaders", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum time gap between the current time and the resolved timestamp in region leaders across TiKV instances. Critical for diagnosing CDC (Change Data Capture) replication lag and identifying regions where transaction resolution is delayed, which can impact downstream data consistency and real-time data synchronization.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572958, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the size distribution of Raft messages exchanged between TiKV peers. Helps diagnose network bandwidth issues and identify abnormally large Raft messages that could impact replication performance and cluster communication efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_peer_msg_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 38, "dashboard": "PD-Cluster-Clinic", "title": "Current storage size", "group": "", "used_for": "Used for monitoring the total storage capacity currently used across the entire TiDB cluster. Helps track storage growth trends and plan capacity expansion before running out of disk space.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 459, "dashboard": "Prometheus-TiCDC-Clinic", "title": "RegionCache operation/s", "group": "KVClient", "used_for": "Used for monitoring the rate of TiCDC region cache operations per second, broken down by operation type. Essential for diagnosing region metadata access patterns and identifying potential bottlenecks in TiCDC's interaction with TiKV's region cache during change data capture.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 471, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resovled ts/s", "group": "", "used_for": "Used for monitoring the rate at which TiKV advances resolved timestamps per second for CDC (Change Data Capture). Critical for diagnosing CDC replication lag and identifying TiKV instances that are slow to advance their resolved timestamps, which can bottleneck downstream data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) " ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 122, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations across different column families (CF) in TiKV's Coprocessor. Helps diagnose query performance issues by identifying which column families are experiencing high scan throughput and detecting potential full table scan problems.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag,cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 400, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV unresolved region count", "group": "", "used_for": "Used for monitoring the number of TiKV regions that CDC (Change Data Capture) has not yet resolved for replication. Critical for diagnosing CDC replication lag and identifying regions where change data capture is blocked or delayed.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12308, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request region count", "group": "Resolved-TS", "used_for": "Used for monitoring the 99th percentile number of regions checked per CheckLeader request in TiKV's Resolved-TS mechanism. Helps identify potential performance bottlenecks when a single CheckLeader request needs to verify an unusually high number of regions, which can impact transaction consistency checks and CDC (Change Data Capture) performance.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 37, "dashboard": "PD-Cluster-Clinic", "title": "Current storage used", "group": "", "used_for": "Used for monitoring the current storage utilization ratio across the TiDB cluster. Helps identify when the cluster is approaching storage capacity limits and needs scaling or data cleanup to prevent storage exhaustion issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 554, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations across different RocksDB column families (CF) in TiKV's coprocessor. Essential for diagnosing index scan performance issues and understanding which column families are handling the most index read traffic during query execution.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag,cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1584, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiKV server overload conditions by tracking scheduler busy errors, channel full events, and coprocessor request rejections. Critical for diagnosing when TiKV instances are overwhelmed by excessive workload, helping identify capacity bottlenecks and the need for scaling or load balancing.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", "sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572000, "dashboard": "TiDB-Cluster-Clinic", "title": "Ingest Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active ingest workers handling global sort operations in TiDB. Helps diagnose performance bottlenecks and resource utilization issues during large-scale data import or DDL operations that require global sorting.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_ingest_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572079, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Leader Resolved TS Region", "group": "Resolved-TS", "used_for": "Used for monitoring the region with the minimum resolved timestamp among all leader regions on each TiKV instance. Critical for identifying lagging regions that may be blocking CDC (Change Data Capture) or stale read progress, helping diagnose replication delays and data consistency issues.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 164, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache OK OPS", "group": "KV Request", "used_for": "Used for monitoring the successful operations per second on TiDB's region cache, broken down by operation type. Helps assess region cache efficiency and identify when cache operations are performing normally versus experiencing failures that could impact query routing performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 251, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client cached regions", "group": "KVClient", "used_for": "Used for monitoring the number of TiKV regions cached by TiCDC's KV client for each changefeed instance. Helps diagnose region cache efficiency and potential memory issues when the cached region count grows abnormally high or drops unexpectedly during data replication.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763571999, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the read throughput rate from cloud storage during TiDB's global sort operations at different percentiles (p50, p95, p999). Essential for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, helping identify slow reads that could impact large-scale data import or DDL operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 220, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS by Type", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded by TiDB to other TiKV nodes, broken down by request type and result status. Essential for diagnosing follower read performance, identifying forwarding bottlenecks, and understanding request routing patterns when TiDB needs to redirect requests to appropriate TiKV replicas.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1615, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages sent from each TiKV instance to other nodes in the cluster. Helps identify network communication patterns, detect instances with abnormal message sending rates, and diagnose Raft replication issues or network bottlenecks between TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 18, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiKV instances, categorized by failure type and store. Critical for diagnosing cluster communication issues, Raft message delivery problems, and identifying unhealthy TiKV nodes that may be causing latency or availability issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7594, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time spent checking memory locks during coprocessor operations in TiKV. Essential for diagnosing performance bottlenecks when concurrent coprocessor requests compete for memory resources, particularly during heavy analytical query workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_mem_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 97, "dashboard": "PD-Cluster-Clinic", "title": "Normal stores", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total number of healthy TiKV stores currently online in the PD cluster. Essential for quickly assessing cluster health and detecting store failures or unavailability that could impact data availability and performance.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_up_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 433, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader region count percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile of region count per check leader request in TiKV instances. Critical for identifying outlier scenarios where TiCDC or other components query an unusually large number of regions in a single check leader operation, which can indicate region distribution imbalances or inefficient scanning patterns.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 453, "dashboard": "Prometheus-TiCDC-Clinic", "title": "ResolveLock count/m", "group": "KVClient", "used_for": "Used for monitoring the rate of lock resolution operations per minute in TiCDC's KV client. Essential for diagnosing transaction conflicts and lock contention issues that may impact CDC replication latency, with different resolution types indicating various conflict scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1452, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report State", "group": "Heartbeat", "used_for": "Used for monitoring the rate and status of bucket reports sent from TiKV stores to PD. Helps diagnose heartbeat communication issues and identify stores with abnormal bucket reporting behavior that could indicate network problems or store health issues.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, type,status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 8379, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock heap size", "group": "Resolved-TS", "used_for": "Used for monitoring the memory consumption of the lock heap in TiKV's Resolved TS module. Helps diagnose memory pressure issues and identify instances with excessive lock accumulation that may impact transaction processing and CDC (Change Data Capture) performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_resolved_ts_lock_heap_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 460, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backoff percentiles", "group": "KVClient", "used_for": "Used for monitoring the 99.999th percentile backoff duration when TiCDC's KV client encounters retryable errors communicating with TiKV. Critical for identifying severe replication delays caused by persistent TiKV access issues, network problems, or region unavailability that trigger exponential backoff mechanisms.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1451, "dashboard": "PD-Cluster-Clinic", "title": "99% Bucket Report Interval", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile interval between bucket heartbeat reports from TiKV stores to PD. Critical for diagnosing communication delays or network issues between TiKV and PD that could affect cluster metadata synchronization and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 250, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache Error OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of region cache errors in TiDB's TiKV client, categorized by error type. Essential for diagnosing region routing issues, stale region metadata problems, and network connectivity failures that can cause query retries and performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"err\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1718, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiKV's Raftstore engine, broken down by instance and error status. Critical for detecting storage layer issues, consensus failures, or I/O problems that could impact data availability and cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 427, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resolved ts fail/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of resolved timestamp advancement failures in TiKV instances, broken down by failure reason. Critical for diagnosing TiCDC replication lag issues and identifying which TiKV nodes are blocking CDC's ability to track data changes in real-time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1616, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft message flush operations per TiKV instance, categorized by flush reason. Helps diagnose Raft communication efficiency and identify instances with abnormal message batching behavior or network transmission issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8377, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observe region status", "group": "Resolved-TS", "used_for": "Used for monitoring the distribution of region resolution statuses in TiKV's Resolved-TS mechanism. Helps diagnose issues with CDC (Change Data Capture) and transaction consistency by tracking which regions are successfully resolving timestamps versus those experiencing delays or failures.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 21, "dashboard": "PD-Cluster-Clinic", "title": "Number of Regions", "group": "", "used_for": "Used for monitoring the total number of leader regions across the PD cluster. Essential for tracking cluster data distribution and identifying potential region imbalances or scheduling issues that could impact I/O performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"leader_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data change capture throughput and identifying bottlenecks in the event collection pipeline from TiKV to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1719, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring the rate of scheduler errors in TiKV, specifically snapshot and prepare write failures. Critical for diagnosing transaction processing issues, storage layer problems, and identifying instances experiencing scheduling failures that could impact write operations.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20, "dashboard": "PD-Cluster-Clinic", "title": "Current peer count", "group": "", "used_for": "Used for monitoring the total number of Raft peer replicas across all regions in the PD cluster. Essential for verifying cluster replication health and detecting issues with replica distribution or missing peers that could impact data availability.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 40, "dashboard": "Dataplane Resource Overview", "title": "Pod Restart Count", "group": "", "used_for": "Used for tracking the cumulative restart count of pods in the dataplane namespace. Essential for identifying unstable pods experiencing crashes or OOM kills, which can indicate resource constraints, configuration issues, or application bugs affecting cluster stability.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "cluster", "control_plane_info", "namespace" ], "promql": [ "sum(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum(kube_pod_container_status_restarts_total{control_plane_info=\"$control_plane_info\",cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 262, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Region Duration", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of loading region information into TiDB's region cache from PD. Critical for diagnosing slow KV request issues caused by region metadata lookup delays, especially when queries need to locate data across TiKV regions.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 399, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table rebalance", "group": "", "used_for": "Used for monitoring the rate of table rebalancing operations across TiCDC instances. Helps identify when tables are being redistributed between capture nodes, which is essential for diagnosing changefeed stability issues and understanding load balancing behavior during scale-out, failover, or manual rebalancing events.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 106, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages received by each TiKV instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster instability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1407, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of read traffic across regions in bytes, tracked through PD's heartbeat mechanism. Helps identify read hotspots and uneven data access patterns that may indicate load balancing issues or the need for region splitting.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 38, "dashboard": "Dataplane Resource Overview", "title": "Pod Last Terminated Reason", "group": "", "used_for": "Used for tracking why pods in the dataplane namespace were last terminated, excluding normal completions. Essential for diagnosing pod crashes, OOM kills, and other abnormal termination events that may indicate resource constraints or application failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", reason!=\"Completed\"}) by (pod,reason)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 11, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Messages", "group": "Raft Message", "used_for": "**Used For:**\n\nUsed for monitoring the rate of different types of Raft messages sent between TiKV nodes. Essential for diagnosing Raft communication issues, network bottlenecks, and understanding cluster consensus behavior during leader elections or region replication.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimate remaining time for initialization", "group": "KVClient", "used_for": "Used for estimating how much time remains for TiCDC's initial region cache population during changefeed startup. Helps operators understand initialization progress and predict when the changefeed will be ready to begin replicating data.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8383, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request size", "group": "Resolved-TS", "used_for": "Used for monitoring the 99th percentile size of CheckLeader requests in TiKV's Resolved-TS mechanism, tracking both byte size and item count. Essential for diagnosing Resolved-TS performance issues and identifying instances with abnormally large CheckLeader requests that could impact transaction consistency guarantees.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 96, "dashboard": "PD-Cluster-Clinic", "title": "Abnormal stores", "group": "", "used_for": "Used for monitoring the health status of TiKV/TiFlash stores in the cluster by tracking disconnected, unhealthy, and low-space stores. Critical for quickly identifying storage node failures, network issues, or capacity problems that could impact cluster availability and data reliability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_disconnected_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_unhealth_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_low_space_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_down_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_offline_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_tombstone_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_slow_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 263, "dashboard": "TiDB-Cluster-Clinic", "title": "RPC Layer Latency", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile network latency of RPC communication between TiDB and TiKV stores. Critical for diagnosing network-related performance issues, identifying slow TiKV nodes, and troubleshooting when queries experience high latency due to network overhead in the KV request path.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store))", "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9166, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fail advance ts count", "group": "Resolved-TS", "used_for": "Used for monitoring failures in advancing the resolved timestamp in TiKV, which is critical for CDC (Change Data Capture) and stale read functionality. Helps diagnose issues with transaction consistency guarantees and identify reasons why resolved timestamps cannot progress, such as stale peer checks or region problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", "sum(delta(tikv_raftstore_check_stale_peer{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1720, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV instances, broken down by error reason. Critical for diagnosing issues with distributed query execution, such as region errors, timeout problems, or data consistency issues that prevent TiDB from successfully processing pushed-down computations.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 472, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader duration percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile latency of TiKV's leader checking operations in TiCDC's resolved timestamp mechanism. Critical for diagnosing tail latency issues in change data capture replication, particularly when TiCDC experiences delays in tracking data changes or leader verification becomes a bottleneck.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2258, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput of TiKV threads, showing the top 20 threads by bytes read/written per second (filtered above 1KB/s). Essential for identifying which threads are performing heavy I/O operations and diagnosing storage bottlenecks or abnormal I/O patterns in TiKV instances.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, sum(rate(tikv_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1411, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of write traffic across regions in the PD cluster by tracking the byte size of region write operations. Helps identify write hotspots, unbalanced region load distribution, and potential performance bottlenecks caused by regions handling disproportionate write traffic.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 99, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event sorting operations in TiCDC's puller component. Essential for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being processed.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8381, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending command size", "group": "Resolved-TS", "used_for": "Used for monitoring the total size of pending commands in TiKV's Resolved TS channel. Helps identify memory pressure and potential bottlenecks in the Resolved TS subsystem when commands accumulate faster than they can be processed.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_resolved_ts_channel_penging_cmd_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1406, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Key", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of region key read operations in PD's scheduler. Helps diagnose region metadata access patterns and identify potential hotspots or bottlenecks in PD's region key reading performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 306, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Hit/Miss OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests that hit or miss the local replica in TiDB. Helps evaluate the effectiveness of stale read optimization and diagnose issues when stale reads frequently miss local data, requiring fallback to leader reads.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 36, "dashboard": "Dataplane Resource Overview", "title": "CPU Throttle", "group": "", "used_for": "Used for monitoring CPU throttling rates of containers in the dataplane namespace. Helps identify when pods are hitting CPU limits and experiencing performance degradation due to resource constraints, indicating potential need for CPU quota adjustments.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 25, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Vote", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft vote messages sent by each TiKV instance during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 42, "dashboard": "Dataplane Resource Overview", "title": "Pod Abnormal Phase", "group": "", "used_for": "Used for identifying pods in abnormal states (not Running or Succeeded) within the specified namespace. Critical for quickly detecting pod failures, crashes, or scheduling issues that require immediate attention in the dataplane cluster.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\",namespace=\"$namespace\",phase!~\"Running|Succeeded\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 307, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests per second in TiDB, broken down by request type. Essential for tracking the usage and throughput of stale read operations, which allow reading slightly outdated data for improved performance and reduced latency in scenarios where strict consistency is not required.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_req_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 727, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Events Flush Lag", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the replication lag between TiCDC receiving change events and successfully flushing them to downstream sinks at different percentiles (p80, p95, p100). Critical for diagnosing data replication delays and identifying performance bottlenecks in the CDC pipeline that could impact downstream data freshness.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.8, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 105, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event merge operations in TiCDC's sorter component. Critical for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being processed.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1721, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV instances, broken down by message type. Critical for identifying communication failures between TiKV and other components (TiDB, PD, or other TiKV nodes), which can indicate network issues, resource exhaustion, or service degradation.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2660, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring threads with high voluntary context switch rates in TiKV, indicating threads frequently yielding CPU due to I/O waits or blocking operations. Helps identify I/O bottlenecks and threads experiencing excessive waiting, which can impact overall TiKV performance and throughput.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, max(rate(tikv_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 200)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "PD-Cluster-Clinic", "title": "Region health", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the health status of regions in the PD cluster, tracking counts of regions in different states (e.g., pending-peer, down-peer, offline-peer, learner-peer). Essential for identifying region scheduling issues, replica availability problems, and cluster stability concerns that may impact data availability and performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1309, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of dropped Raft consensus messages in TiKV, categorized by message type. Critical for diagnosing cluster communication issues, network problems, or overloaded nodes that prevent proper Raft replication and consensus.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1722, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region transfers or drops on each TiKV instance. Critical for diagnosing cluster instability, network partitions, or load balancing issues that cause frequent leader changes and potential service disruptions.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "PD-Cluster-Clinic", "title": "Leader/Primary", "group": "", "used_for": "Used for identifying the current leader/primary member in the PD (Placement Driver) cluster. Essential for verifying cluster leadership status, troubleshooting split-brain scenarios, and confirming which PD instance is actively managing cluster metadata and scheduling decisions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "service_member_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2108, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the distribution of thread states (running, sleeping, waiting, etc.) across TiKV instances. Essential for diagnosing thread contention, resource saturation, and identifying instances with abnormal thread behavior that may indicate performance bottlenecks or deadlock situations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2661, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for identifying TiKV threads experiencing excessive involuntary context switches (>100/sec), which indicates CPU contention or resource starvation. Critical for diagnosing performance degradation caused by thread scheduling issues, where threads are forcibly preempted by the OS rather than yielding voluntarily.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, max(rate(tikv_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 100)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 19, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring the uptime duration of TiCDC, TiKV, and PD components since their last restart. Essential for tracking service availability, identifying unexpected restarts, and verifying cluster stability after maintenance or deployments.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tikv\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3638, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes across TiKV instances to understand data distribution patterns. Helps identify region size imbalances, oversized regions that may need splitting, or undersized regions that could indicate excessive splitting, which can impact I/O performance and cluster efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1412, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Key", "group": "Heartbeat distribution ", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of keys written per region during PD's scheduling operations. Helps diagnose region split/merge performance and identify regions with unbalanced write workloads that may cause hotspots or scheduling bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 53, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting operations in TiCDC's puller entry sorter. Critical for diagnosing replication lag issues caused by slow DDL processing, especially when changefeeds experience delays in handling schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573729, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Allocator Stats", "group": "Memory", "used_for": "Used for monitoring memory allocator statistics in TiKV instances, tracking memory allocation patterns and usage across different allocator components. Essential for diagnosing memory-related issues, memory leaks, and understanding TiKV's memory consumption behavior at the allocator level.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_allocator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1449, "dashboard": "PD-Cluster-Clinic", "title": "Offline store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store removal operations in PD when taking TiKV nodes offline. Helps track decommissioning status and estimate completion time when scaling down the cluster or replacing failed nodes.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"removing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 26, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Goroutine count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of goroutines and OS threads in TiCDC instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate performance degradation or stability issues in the change data capture service.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 106, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting and merging operations in TiCDC's puller component. Critical for diagnosing replication lag issues caused by slow DDL event processing, especially when changefeeds experience delays in synchronizing schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1408, "dashboard": "PD-Cluster-Clinic", "title": "Store Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of heartbeat intervals between TiKV stores and PD. Helps diagnose network issues, store health problems, or PD scheduling delays when heartbeat intervals become irregular or exceed expected thresholds.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1723, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions missing their leader on each TiKV instance. Critical for detecting split-brain scenarios, network partitions, or leader election failures that can cause service unavailability and data access issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "Dataplane Resource Overview", "title": "Nodes", "group": "Node", "used_for": "Used for monitoring the distribution of Kubernetes nodes by instance type in the dataplane cluster. Helps operators understand cluster composition and identify resource allocation patterns across different node types.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_beta_kubernetes_io_instance_type)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 76, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft administrative operations in TiKV, specifically configuration changes and leader transfers. Essential for diagnosing cluster rebalancing activities, troubleshooting region migration issues, and understanding the frequency of topology changes during scaling or maintenance operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 138, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation efficiency in TiKV by tracking where data is retrieved from (memtable, block cache, or L0 SST files). Essential for diagnosing read performance issues and understanding cache hit rates versus disk I/O patterns in the storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573730, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "group": "Memory", "used_for": "Used for monitoring the net memory allocation rate per thread in TiKV, showing allocated bytes (positive) minus released bytes (negative). Essential for identifying memory leaks, understanding per-thread memory pressure, and diagnosing which threads are accumulating memory over time.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name) - sum(rate(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"dealloc\"}[$__rate_interval])) by (thread_name) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 308, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req Traffic", "group": "KV Request", "used_for": "Used for monitoring the network traffic volume of stale read requests in TiDB, broken down by result status and direction (sent/received). Essential for tracking the I/O impact of stale read operations and diagnosing performance issues when using follower reads or historical data queries.", "component": "TIDB", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, direction)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 26, "dashboard": "Dataplane Resource Overview", "title": "Node Load 1m", "group": "Node", "used_for": "Used for monitoring the 1-minute load average per CPU core across cluster nodes. Helps identify nodes experiencing high system load or resource contention, which can indicate performance bottlenecks or capacity issues requiring investigation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_load1_per_cpu:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1453, "dashboard": "PD-Cluster-Clinic", "title": "Online store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store preparation operations in PD, specifically tracking stores that are in the \"preparing\" state during online operations. Helps diagnose store initialization issues and identify stores that may be stuck during the onboarding process to the cluster.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"preparing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second (OPS) of different PD client commands from TiDB instances, excluding TSO requests. Helps identify which PD API calls (like GetRegion, GetStore) are most frequently used and diagnose potential bottlenecks in TiDB's communication with PD.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"tso\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1409, "dashboard": "PD-Cluster-Clinic", "title": "Region Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of time intervals between consecutive region heartbeats sent to PD. Essential for detecting heartbeat irregularities, network issues, or TiKV instances that are slow to report region status, which can impact cluster metadata accuracy and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763571998, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of reading data from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, particularly when investigating slow data import or index building operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 27, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Open FD count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiCDC processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiCDC to fail or become unstable.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2696, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Newly Allocated Bytes by Thread", "group": "Memory", "used_for": "Used for tracking memory allocation rates across different TiKV threads. Helps identify which threads are consuming the most memory and diagnose memory-related performance issues or potential memory leaks in specific thread operations.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 705, "dashboard": "Prometheus-TiCDC-Clinic", "title": "batch read sorter size percentile", "group": "KVClient", "used_for": "Used for monitoring the 99.9th percentile batch size when TiCDC's sorter reads data from storage. Critical for diagnosing data processing bottlenecks and identifying when the sorter is handling unusually large batches that could impact replication latency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,instance,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1111, "dashboard": "PD-Cluster-Clinic", "title": "Left time", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the estimated time remaining for PD cluster operations such as region scheduling, leader balancing, or other ongoing cluster adjustments. Helps operators understand when current rebalancing or migration tasks are expected to complete, enabling better planning for maintenance windows and capacity changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1454, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of bucket report intervals from TiKV stores to PD. Helps diagnose heartbeat delays and communication issues between TiKV and PD, which can indicate network problems or overloaded stores affecting cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763572510, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Damaged files", "group": "Errors", "used_for": "Used for monitoring corrupted files detected in TiKV's RocksDB storage engine and tracking how many damaged files have been deleted. Critical for identifying data integrity issues and potential disk corruption that could lead to data loss or cluster instability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_rocksdb_damaged_files{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "tikv_rocksdb_damaged_files_deleted{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572588, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Log Replication Rejected", "group": "Errors", "used_for": "Used for monitoring the rate of rejected Raft log append operations in TiKV instances. Critical for identifying replication bottlenecks, resource constraints, or configuration issues that prevent followers from accepting log entries from the leader.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 82, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB Get operation latency in TiKV, tracking maximum, P99, and P95 durations. Essential for diagnosing point lookup performance issues and identifying when read operations from the storage engine are experiencing slowdowns or outliers.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft Admin", "used_for": "Used for monitoring the rate of successful Raft admin commands applied in TiKV, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes, helping diagnose cluster topology changes and Raft group management issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Usage", "group": "Node", "used_for": "Used for monitoring CPU utilization across individual nodes in the dataplane cluster. Helps identify nodes experiencing high CPU load, enabling quick detection of resource bottlenecks and uneven workload distribution across the infrastructure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_cpu_utilisation:rate1m{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 129, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation rates in TiKV, tracking total seeks, successful seeks, and iterator next operations. Essential for diagnosing read performance issues and identifying inefficient scan patterns that may indicate missing indexes or suboptimal query execution plans.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1467, "dashboard": "PD-Cluster-Clinic", "title": "Hot Degree of Bucket", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of hot bucket degrees in PD's scheduler, tracking how data hotspots are distributed across buckets over time. Essential for diagnosing load balancing issues and identifying regions with concentrated read/write traffic that may cause performance bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 706, "dashboard": "Prometheus-TiCDC-Clinic", "title": "batch read sorter duration percentile", "group": "KVClient", "used_for": "Used for monitoring the 99.9th percentile latency of batch read operations in TiCDC's sorter component. Critical for diagnosing data replication delays and identifying performance bottlenecks in the change data capture pipeline when reading sorted events.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573731, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Recently Released Bytes by Thread", "group": "Memory", "used_for": "Used for tracking memory deallocation patterns across different TiKV threads. Helps identify which threads are releasing memory and diagnose potential memory leaks or inefficient memory management when deallocation rates are abnormally low or imbalanced.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 71, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested CPU resources to allocatable CPU capacity on each Kubernetes node. Helps identify nodes approaching resource limits and potential scheduling constraints that could prevent new pod deployments.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 70, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check split", "group": "Raft Admin", "used_for": "Used for monitoring the rate of region split checks in TiKV's Raftstore, categorized by check type. Helps diagnose region split behavior and identify potential issues with automatic region splitting when regions grow too large or experience high load.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of PD client commands (excluding TSO operations) from TiDB instances at p90, p99, and p999 percentiles. Essential for diagnosing communication delays between TiDB and PD, helping identify network issues, PD overload, or metadata operation bottlenecks that impact query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 125, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiKV, tracking maximum, P99, and P95 durations. Essential for diagnosing read performance issues caused by inefficient data scanning or index lookups in the storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1112, "dashboard": "PD-Cluster-Clinic", "title": "Current scaling speed", "group": "", "used_for": "Used for monitoring the current scaling speed of the PD cluster, indicating how fast the cluster is performing data rebalancing and region scheduling operations. Essential for tracking the progress of cluster scaling activities and diagnosing whether rebalancing is proceeding at expected rates during capacity changes or maintenance operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 43, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed PD client commands from TiDB instances, categorized by command type. Essential for identifying communication issues between TiDB and PD, helping diagnose cluster metadata access problems, TSO allocation failures, or network connectivity issues that could impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 707, "dashboard": "Prometheus-TiCDC-Clinic", "title": "mounter group channel size", "group": "KVClient", "used_for": "Used for monitoring the input channel buffer size of TiCDC's mounter group, which processes raw KV change events before converting them to table-level changes. Critical for identifying backpressure issues when the channel fills up, indicating that the mounter cannot keep pace with incoming KV events from TiKV.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 24, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CPU usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiCDC server processes and comparing it against the configured CPU limit (GOMAXPROCS). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiCDC needs additional CPU resources to handle replication workload efficiently.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}[2m])", "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 30, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Usage", "group": "Node", "used_for": "Used for monitoring memory utilization ratio across individual nodes in the dataplane cluster. Helps identify nodes experiencing memory pressure or imbalanced memory consumption that could impact workload performance and stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_memory_utilisation:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 71, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft Admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiKV's Raftstore. Critical for diagnosing extreme tail latency in region split operations, which can indicate issues with region management, hotspot handling, or cluster scaling performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiCDC server memory consumption, tracking both resident memory (RSS) and Go heap allocation. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC resource allocation during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3924, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU consumption by TiKV backup worker threads and I/O operations during backup tasks. Essential for identifying backup performance bottlenecks, detecting resource contention during backup operations, and ensuring backups don't excessively impact cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"b.*k.*w.*k.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_io\"}[2m])) by (instance)", "tikv_backup_softlimit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 33, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "Used for monitoring the total storage size of each RocksDB column family (CF) across TiKV instances. Essential for tracking storage capacity usage, identifying which column families are consuming the most disk space, and detecting abnormal storage growth that may indicate data bloat or compaction issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "PD-Cluster-Clinic", "title": "Placement Rules Status", "group": "", "used_for": "Used for monitoring the status of PD's placement rule manager, which controls data placement policies across the TiKV cluster. Essential for verifying that placement rules are properly configured and active, helping diagnose issues with data distribution, replica placement, and label-based scheduling constraints.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_rule_manager_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 113, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership", "group": "Server", "used_for": "Used for monitoring the rate of ownership changes in TiCDC, tracking how frequently the owner role transfers between TiCDC nodes. Essential for diagnosing cluster stability issues, as frequent ownership changes may indicate node failures, network problems, or configuration issues affecting TiCDC's high availability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572060, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split duration", "group": "Raft Admin", "used_for": "Used for monitoring the latency of load-based region split operations in TiKV at the 80th and 90th percentiles. Essential for diagnosing performance issues when TiKV automatically splits regions based on load, helping identify bottlenecks in region rebalancing and hotspot mitigation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.90, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_load_base_split_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiKV's RocksDB storage engine, including successful writes (self/other), timeout occurrences, and WAL writes. Essential for diagnosing write throughput issues, identifying write bottlenecks, and detecting abnormal write timeout patterns that may indicate storage performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 112, "dashboard": "PD-Cluster-Clinic", "title": "Syncer index", "group": "Region storage", "used_for": "Used for monitoring the synchronization index of PD's region syncer, which tracks the progress of region metadata synchronization across PD nodes. Essential for diagnosing region information consistency issues and detecting synchronization lag that could affect cluster metadata accuracy.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"sync_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 77, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO Wait Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the wait duration (p90, p99, p999) when TiDB requests TSO (Timestamp Oracle) from PD. Critical for diagnosing transaction latency issues caused by PD client bottlenecks or network delays between TiDB and PD servers.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 72, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested memory resources to allocatable memory capacity on each Kubernetes node. Helps identify nodes approaching memory resource limits and potential scheduling constraints before pods fail to deploy due to insufficient memory requests.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",unit=\"byte\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3926, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of backup threads allocated in each TiKV instance's thread pool. Essential for diagnosing backup performance issues and ensuring adequate thread resources are available during backup operations, especially when backups are slow or timing out.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO OPS", "group": "PD Client", "used_for": "Used for monitoring the throughput of TSO (Timestamp Oracle) requests from TiDB to PD, measured in operations per second. Essential for identifying TSO request bottlenecks, understanding transaction workload intensity, and diagnosing issues when TiDB experiences timestamp allocation delays that could impact transaction performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))", "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 22, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the rate of channel full events in TiKV instances, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, thread pool saturation, or processing bottlenecks that can lead to request timeouts and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 126, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB write operation latency in TiKV, displaying maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or storage engine contention.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 293, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD leader history", "group": "Server", "used_for": "**Used For:**\n\nUsed for tracking PD leader election history and identifying leadership changes in the cluster. Essential for diagnosing cluster stability issues, investigating unexpected failovers, and correlating performance anomalies with PD leader switches.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 110, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership history", "group": "Server", "used_for": "Used for tracking TiCDC owner role changes and leadership transfers across instances. Helps diagnose cluster stability issues, frequent leader elections, or ownership handoff problems that could impact replication continuity.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 289, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd MVCC DB total size", "group": "Etcd", "used_for": "Used for monitoring the total size of PD's embedded etcd MVCC database in bytes. Critical for tracking metadata storage growth and identifying potential disk space issues or excessive metadata accumulation that could impact cluster stability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3636, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split event", "group": "Raft Admin", "used_for": "**Used For:**\n\nUsed for monitoring the rate of load-based region split events in TiKV, categorized by split type. Helps diagnose whether TiKV is actively splitting regions due to load pressure (hot spots) and evaluate the effectiveness of automatic load balancing mechanisms.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum(delta(tikv_load_base_split_event[1m])) by (type)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 116, "dashboard": "PD-Cluster-Clinic", "title": "PD scheduler config", "group": "Cluster", "used_for": "Used for monitoring the configuration status of PD schedulers in the cluster. Helps verify scheduler settings and diagnose issues related to region balancing, leader distribution, and other scheduling behaviors that may affect cluster performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 54, "dashboard": "Dataplane Resource Overview", "title": "Root Disk Usage", "group": "Node", "used_for": "**Used For:**\n\nUsed for monitoring root filesystem disk space utilization across cluster nodes. Essential for identifying nodes approaching disk capacity limits, which can cause service disruptions, write failures, or system instability if storage becomes full.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "1 - sum(node_filesystem_avail_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device) / sum(node_filesystem_size_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 78, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO RPC Duration", "group": "PD Client", "used_for": "Used for monitoring the tail latency of TSO (Timestamp Oracle) RPC requests from TiDB to PD at p99, p999, and p9999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication issues that impact overall cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99, 999, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 113, "dashboard": "PD-Cluster-Clinic", "title": "History last index", "group": "Region storage", "used_for": "Used for monitoring the last Raft log index in PD's region synchronization process. Helps diagnose region metadata synchronization issues and detect potential delays or inconsistencies in region state replication across PD nodes.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"last_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 137, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations per second in TiKV's RocksDB engine. Critical for diagnosing write durability overhead and identifying potential I/O bottlenecks when fsync operations become a performance constraint.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the rate of regions with active write operations on each TiKV instance. Helps identify write hotspots and uneven write distribution across the cluster, which can indicate load balancing issues or hotspot regions requiring attention.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573619, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer in Flashback State", "group": "Raft Admin", "used_for": "Used for monitoring the number of Raft peers currently in flashback state on each TiKV instance. Essential for tracking the progress and scope of flashback operations, helping diagnose issues when restoring the cluster to a previous point in time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_peer_in_flashback_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 114, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd health check duration", "group": "Etcd", "used_for": "Used for monitoring the latency of TiCDC's health check operations against etcd at p95, p99, and p999 percentiles. Critical for diagnosing etcd connectivity issues and detecting when TiCDC's coordination layer experiences slow responses, which can impact change data capture reliability and cluster metadata operations.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 130, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of writing to RocksDB's Write-Ahead Log (WAL) in TiKV. Critical for diagnosing disk I/O performance issues and identifying storage bottlenecks that impact write operations, as high WAL write duration directly affects transaction commit latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5264, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Errors", "group": "Backup & Import", "used_for": "Used for monitoring the rate of backup operation errors in TiKV instances over 2-minute intervals. Essential for quickly identifying backup failures, diagnosing data protection issues, and ensuring backup reliability in the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "cloud request", "group": "Backup & Import", "used_for": "Used for monitoring the rate of cloud storage requests (such as S3 or GCS operations) made by TiKV during backup and restore operations. Helps identify cloud API call patterns and diagnose issues with external storage interactions during data import/export tasks.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_cloud_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cloud, req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3637, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TopN QPS exceeds threshold", "group": "Raft Admin", "used_for": "Used for monitoring when the top read query throughput in TiKV exceeds predefined thresholds. Helps identify hot read regions or specific queries causing excessive load, enabling quick detection of unbalanced traffic distribution or potential hotspot issues in the cluster.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "tikv_read_qps_topn{order=\"0\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 159, "dashboard": "TiDB-Cluster-Clinic", "title": "Async TSO Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of asynchronous TSO (Timestamp Oracle) requests from TiDB to PD at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication bottlenecks that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6, "dashboard": "Dataplane Resource Overview", "title": "Deployment", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for each deployment in the selected namespace. Helps verify deployment scaling configurations and identify discrepancies between intended and actual replica counts during capacity planning or troubleshooting deployment issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_deployment_spec_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed table count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being replicated by each TiCDC instance within a changefeed. Helps identify table distribution imbalances across TiCDC nodes and diagnose replication coverage issues when troubleshooting data synchronization problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 58, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region across TiKV instances. Helps identify uneven write distribution, hotspot regions, and instances experiencing abnormally high or low write activity that may indicate load imbalance or performance issues.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 264, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the p99 and p999 latency of TiCDC's etcd worker tick reactor operations. Essential for diagnosing TiCDC replication delays caused by slow etcd interactions, particularly when investigating change feed lag or metadata synchronization issues.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 267, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the latency distribution of TiCDC's etcd worker tick reactor operations. Essential for diagnosing performance bottlenecks in TiCDC's etcd interaction layer, particularly when experiencing replication delays or coordination issues with the etcd cluster.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 139, "dashboard": "PD-Cluster-Clinic", "title": "Cluster ID", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for displaying the unique cluster ID of the PD (Placement Driver) cluster. Helps verify cluster identity and distinguish between multiple TiDB clusters in multi-cluster environments or during troubleshooting.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 256, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction execution duration in TiCDC's etcd worker. Critical for diagnosing coordination and metadata operation latency issues that could impact TiCDC's change data capture performance and replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1473, "dashboard": "PD-Cluster-Clinic", "title": "State", "group": "DR Autosync", "used_for": "Used for monitoring the current state of PD's disaster recovery (DR) auto-sync replication. Essential for tracking DR replication status and quickly identifying when the cluster transitions between sync states (e.g., sync, async, or disconnected) during disaster recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=\"$instance\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1433, "dashboard": "PD-Cluster-Clinic", "title": "Store limit", "group": "Cluster", "used_for": "Used for monitoring the configured rate limits on TiKV stores, which control the maximum number of operations (like adding/removing peers) allowed per store. Essential for diagnosing scheduling throttling issues and understanding why certain stores may not be accepting new regions or replicas as expected.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [], "promql": [ "pd_cluster_store_limit" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 103, "dashboard": "PD-Cluster-Clinic", "title": "Region label isolation level", "group": "Cluster", "used_for": "Used for monitoring the distribution of regions across different label isolation levels in PD's placement rules. Helps verify that regions are properly distributed according to configured label constraints (e.g., rack, zone, host) for high availability and fault tolerance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 222, "dashboard": "TiDB-Cluster-Clinic", "title": "Request forwarded status", "group": "PD Client", "used_for": "Used for monitoring whether TiDB's PD client requests are being forwarded to other PD nodes instead of being handled directly by the connected PD leader. Helps diagnose PD leadership changes, network routing issues, or PD cluster instability that causes request redirection overhead.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "pd_client_request_forwarded_status" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3646, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the total rate of bytes written to regions across all TiKV instances. Helps identify write-heavy workloads, diagnose uneven data distribution, and detect sudden increases in write traffic that may impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3927, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST files being backed up from TiKV's write column family. Helps diagnose backup performance issues and identify whether large SST files are causing backup slowdowns or bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1481, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the distribution of Region sizes in TiKV, showing p95, p99, and average values. Essential for identifying Region size imbalances, detecting abnormally large Regions that may cause hotspots or split issues, and ensuring optimal data distribution across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1478, "dashboard": "PD-Cluster-Clinic", "title": "State of all instances", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync state of all PD instances in the cluster. Essential for verifying DR replication status and quickly identifying instances that may have fallen out of sync or encountered replication issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 4, "dashboard": "Dataplane Resource Overview", "title": "StatefulSet", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for StatefulSets in the selected namespace. Helps verify StatefulSet scaling configurations and detect mismatches between desired and actual replica counts during deployment or scaling operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 642, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture tables count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being captured and replicated by each TiCDC changefeed instance. Helps identify table distribution imbalances across changefeeds and diagnose replication coverage issues when tables are unexpectedly missing from capture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 128, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiKV, broken down by operation type (flush, compaction levels). Essential for diagnosing storage engine performance issues, identifying excessive compaction activity that may impact write throughput, and understanding background I/O patterns in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5266, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in the default column family during backup operations. Helps identify backup performance issues and diagnose whether large SST files are causing slow backup speeds or resource bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 258, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration percentile", "group": "Etcd", "used_for": "Used for monitoring the p95 and p99 latency of etcd transaction executions in TiCDC's etcd worker. Critical for diagnosing metadata operation performance issues and identifying when etcd transactions are experiencing delays that could impact TiCDC's change data capture operations.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12882, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write duration", "group": "Raft Log", "used_for": "Used for monitoring the write latency (p95, p99) when TiKV garbage collects Raft logs to disk. Critical for diagnosing performance issues related to Raft log cleanup operations, especially when log accumulation or slow disk writes impact cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20, "dashboard": "Dataplane Resource Overview", "title": "DaemonSet", "group": "Pod", "used_for": "Used for monitoring the number of scheduled pods for each DaemonSet in the selected namespace. Helps verify that DaemonSets are properly deployed across nodes and identify scheduling issues when the actual pod count doesn't match expected node coverage.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_daemonset_status_current_number_scheduled{cluster=\"$cluster\",namespace=\"$namespace\"}) by (daemonset)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1474, "dashboard": "PD-Cluster-Clinic", "title": "Tick", "group": "DR Autosync", "used_for": "Used for monitoring the rate of DR (Disaster Recovery) auto-sync ticks in PD. Helps verify that the DR auto-sync mechanism is actively running and processing synchronization cycles, with abnormal rates indicating potential issues with the DR replication coordination.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=\"$instance\"}[5m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 117, "dashboard": "PD-Cluster-Clinic", "title": "Label distribution", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of placement labels across the TiDB cluster managed by PD. Helps diagnose data placement issues, verify label configuration correctness, and ensure proper data distribution according to placement rules for high availability and disaster recovery.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 57, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region across TiKV instances. Helps identify unbalanced write workloads, hotspot regions, and assess whether data distribution is optimal for write performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 135, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the latency of Write-Ahead Log (WAL) file synchronization operations in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues, as high WAL sync latency directly impacts transaction commit times and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 115, "dashboard": "PD-Cluster-Clinic", "title": "Current ID allocation", "group": "Cluster", "used_for": "Used for monitoring the current maximum ID allocation value in the PD cluster. Helps track ID allocation progress and diagnose potential ID exhaustion issues or abnormal allocation patterns that could impact cluster operations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"idalloc\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 246, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of HTTP requests from TiDB to PD (Placement Driver) at different percentiles (p90, p99, p999). Essential for diagnosing communication issues between TiDB and PD, identifying network bottlenecks, or PD performance problems that could impact metadata operations and transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2453, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction guard actions", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of compaction guard actions in TiKV's RocksDB storage engine across default and write column families. Essential for diagnosing write stalls and understanding how TiKV is managing compaction pressure to prevent performance degradation during heavy write workloads.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_compaction_guard_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=~\"default|write\"}[2m])) by (cf, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 247, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of HTTP API requests from TiDB servers to PD (Placement Driver). Helps identify communication patterns between TiDB and PD, diagnose connectivity issues, and detect abnormal request spikes that may indicate metadata access problems or cluster coordination issues.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 254, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction sizes in TiCDC's etcd worker component. Helps identify abnormally large transactions that could impact etcd performance and diagnose potential issues with TiCDC's metadata operations or changefeed state management.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3929, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the rate at which TiKV generates SST (Sorted String Table) files during backup operations, measured in bytes per second. Essential for tracking backup performance, identifying throughput bottlenecks, and diagnosing slow backup issues across the cluster or specific instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 86, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint", "group": "Changefeed", "used_for": "Used for monitoring TiCDC changefeed replication progress by comparing the checkpoint timestamp against the current cluster TSO (timestamp oracle). Essential for identifying replication lag, detecting stuck changefeeds, and understanding barriers that may be blocking data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_owner_barrier_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 248, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed HTTP requests from TiDB to PD (Placement Driver). Essential for identifying communication issues between TiDB and PD, such as network problems, PD unavailability, or API errors that could impact cluster metadata operations and query routing.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 5597, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying bottlenecks when creating cluster snapshots.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3647, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the rate of keys written per second across all regions in TiKV instances. Helps identify write hotspots, uneven data distribution, and diagnose performance issues related to region-level write activity.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12881, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection write operations per TiKV instance. Helps identify instances with high log cleanup activity, which may indicate heavy write workloads or potential performance bottlenecks in log management.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12886, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC kv sync duration", "group": "Raft Log", "used_for": "Used for monitoring the latency of syncing Raft logs to the underlying key-value storage engine in TiKV, tracking p95 and p99 percentiles. Critical for diagnosing write performance bottlenecks and disk I/O issues that affect Raft log persistence and cluster consensus stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dataplane Resource Overview", "title": "Pod Status", "group": "Pod", "used_for": "Used for monitoring the current status phase of pods in the specified namespace. Helps quickly identify pod health issues by showing which pods are in Running, Pending, Failed, or other phases, enabling rapid detection of deployment or scheduling problems.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1427, "dashboard": "PD-Cluster-Clinic", "title": "CPU Usage", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of PD (Placement Driver) instances against their configured CPU limits (GOMAXPROCS). Essential for identifying CPU bottlenecks, resource contention, or when PD processes are approaching their maximum CPU capacity, which can impact cluster scheduling and metadata operations.", "component": "PD", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}[2m])", "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1477, "dashboard": "PD-Cluster-Clinic", "title": "State ID", "group": "DR Autosync", "used_for": "Used for monitoring the current disaster recovery (DR) state identifier in PD's auto-sync replication. Essential for tracking DR state transitions and verifying the cluster's replication status during failover or recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1475, "dashboard": "PD-Cluster-Clinic", "title": "Recovered region", "group": "DR Autosync", "used_for": "Used for monitoring the number of regions that have been recovered in the DR (Disaster Recovery) auto-sync process. Critical for tracking recovery progress after a disaster event and verifying that data replication has been successfully restored across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recovered_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 50, "dashboard": "Dataplane Resource Overview", "title": "Memory Utilization", "group": "Pod", "used_for": "Used for monitoring memory utilization percentage of pods in the dataplane by comparing actual memory usage (working set) against configured resource limits. Helps identify pods approaching memory limits, potential OOM risks, and opportunities for resource allocation optimization.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\"})by(pod)/\nsum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}\n)\n/\nsum by (pod) (\n kube_pod_container_resource_limits{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"POD\"}\n)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 136, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiKV, showing maximum, P99, and P95 durations. Critical for diagnosing write amplification issues, storage performance degradation, and identifying when background compaction operations are impacting foreground query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 512, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts", "group": "Changefeed", "used_for": "Used for monitoring the replication progress of TiCDC changefeeds by comparing the resolved timestamp (the point up to which data has been replicated) against the current PD cluster TSO. Essential for identifying replication lag, detecting slow tables that are holding back the changefeed, and diagnosing data synchronization delays in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3931, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying bottlenecks in the data scanning phase of backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 260, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size percentile", "group": "Etcd", "used_for": "Used for monitoring the 95th and 99th percentile transaction size in bytes for TiCDC's etcd worker operations. Essential for identifying abnormally large etcd transactions that could impact TiCDC's metadata management performance and detecting potential issues with change data capture coordination overhead.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with real-time changes, helping diagnose replication bottlenecks or downstream processing issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12884, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC seek operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of seek operations performed during Raft log garbage collection in TiKV. Helps identify I/O overhead from log cleanup activities and diagnose performance issues related to excessive seek operations during Raft log GC processes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_seek_operations_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1476, "dashboard": "PD-Cluster-Clinic", "title": "Recover progress", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync recovery progress in PD. Essential for tracking the completion status when restoring data synchronization after a DR event or cluster failure, helping operators understand how far along the recovery process has progressed.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recover_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 315, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB Wait TSO Future Duration", "group": "PD Client", "used_for": "Used for monitoring the duration TiDB waits when a TSO (timestamp oracle) timestamp is in the future, tracking p90, p99, and p999 latencies. Critical for diagnosing clock synchronization issues between TiDB and PD servers, which can cause transaction delays and performance degradation when system clocks are skewed.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_ts_future_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le) / sum(rate(tidb_tikvclient_ts_future_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 140, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file reads in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing storage layer performance issues, especially when queries experience slow disk I/O or compaction-related delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3720, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of requests received by TiKV servers, showing both average and 99th percentile values by request type. Helps diagnose request batching efficiency and identify whether clients are sending requests individually or in optimized batches, which impacts network overhead and processing performance.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "Dataplane Resource Overview", "title": "CPU Utilization", "group": "Pod", "used_for": "Used for monitoring CPU utilization of pods in the dataplane, showing both actual usage against limits and resource requests against node capacity. Essential for identifying CPU bottlenecks, over-provisioned pods, and node resource pressure that could impact cluster performance.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)/\nsum(kube_pod_container_resource_limit_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\"}[$__rate_interval]\n ) and on (container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)\n/\nsum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)", "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1429, "dashboard": "PD-Cluster-Clinic", "title": "Memory Usage", "group": "Cluster", "used_for": "Used for monitoring memory consumption of PD (Placement Driver) instances, including process resident memory, Go heap system memory, and heap in-use memory. Essential for detecting memory leaks, identifying OOM risks, and diagnosing PD stability issues caused by excessive memory usage.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3730, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Hibernate Peers", "group": "Server", "used_for": "Used for monitoring the count and state of hibernated Raft peers in TiKV instances. Helps diagnose region management efficiency and identify potential issues with peer hibernation mechanisms that optimize resource usage for idle regions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_hibernated_peer_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 513, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays in changefeeds, helping diagnose slow replication issues and ensuring downstream systems receive timely data updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12887, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log lag", "group": "Raft Log", "used_for": "**Used For:**\n\nUsed for monitoring the lag between Raft log replication and application in TiKV instances. Critical for identifying replication delays that can indicate slow followers, network issues, or overloaded nodes affecting data consistency and cluster health.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_log_lag_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 291, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% WAL fsync duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile fsync duration of etcd's Write-Ahead Log (WAL) in TiCDC. Critical for diagnosing disk I/O performance issues that could impact TiCDC's metadata operations and change data capture reliability, as high WAL fsync latency indicates slow disk writes affecting etcd's consistency guarantees.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12975, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log gc skipped", "group": "Raft Log", "used_for": "Used for monitoring the rate of skipped Raft log garbage collection operations in TiKV, broken down by instance and skip reason. Essential for diagnosing issues where Raft logs aren't being cleaned up properly, which can lead to excessive disk usage and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_skipped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 290, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% Handle transactions duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in TiCDC's coordination layer. Critical for diagnosing slow metadata operations and replication coordination issues that could impact TiCDC's change data capture performance.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1430, "dashboard": "PD-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring how long each PD instance has been running since its last restart. Essential for tracking PD stability, identifying recent restarts or crashes, and verifying cluster continuity during troubleshooting or maintenance operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Requests", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring memory resource requests and limits configured for pods in the dataplane namespace. Helps identify pod memory allocation settings and detect potential resource constraint issues when actual usage approaches or exceeds configured requests/limits.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 2451, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency and reasons why RocksDB triggers compaction operations in TiKV. Helps diagnose storage engine behavior and identify whether compactions are triggered by normal operations (like level size limits) or abnormal conditions (like too many files), which is critical for understanding write amplification and I/O performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing slow DDL execution, schema synchronization delays, and identifying instances experiencing performance issues when fetching or updating table metadata from TiKV.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1431, "dashboard": "PD-Cluster-Clinic", "title": "Goroutine Count", "group": "Cluster", "used_for": "Used for monitoring the number of active goroutines in PD instances. Helps identify goroutine leaks, resource exhaustion issues, or abnormal concurrency patterns that could indicate PD performance problems or potential memory issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3928, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, tracking p95, p99, and average durations. Critical for diagnosing backup performance issues and identifying slow backup ranges that may impact backup completion times or cluster performance during backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 102, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the block cache memory usage across different column families in TiKV's RocksDB storage engine. Essential for diagnosing memory pressure issues, cache efficiency problems, and optimizing RocksDB block cache configuration to balance memory usage and read performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 603, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Difference of resolvedTs and checkpointTs", "group": "Changefeed", "used_for": "Used for monitoring the replication lag in TiCDC changefeeds by measuring the time difference between resolvedTs (latest transaction timestamp processed) and checkpointTs (latest timestamp successfully replicated downstream). Critical for identifying replication delays and ensuring data consistency between upstream TiDB and downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "(max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed) - max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)) / 1000 > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Usage", "group": "Pod", "used_for": "Used for monitoring the working set memory usage of individual pods in the TiDB cluster. Helps identify memory-intensive pods, detect memory leaks, and diagnose out-of-memory (OOM) issues before they impact cluster stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 7266, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory trace", "group": "Server", "used_for": "Used for tracking detailed memory usage breakdown in TiKV, specifically monitoring memory consumption of Raftstore threads and Raft Engine components. Essential for diagnosing memory-related issues, identifying memory leaks, and understanding which TiKV subsystems are consuming the most memory resources.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}", "raft_engine_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12974, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC failed", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection failures in TiKV instances. Critical for identifying storage cleanup issues that can lead to disk space exhaustion and degraded cluster performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_failed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9560, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Entry Cache Evicts", "group": "Server", "used_for": "Used for monitoring the rate at which Raft log entries are evicted from TiKV's in-memory cache. High eviction rates indicate memory pressure or insufficient cache size, which can impact Raft replication performance and increase disk I/O for log retrieval.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_raft_entries_evict_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the hit rate of RocksDB memtable lookups in TiKV. Helps diagnose read performance by showing how effectively recent writes are being served from memory before flushing to disk, with low hit rates potentially indicating memory pressure or suboptimal memtable configuration.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 29, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Lease Error OPM", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema lease errors per minute in TiDB instances. Critical for detecting schema synchronization issues between TiDB servers, which can cause query failures or inconsistent metadata when the schema lease mechanism fails to maintain consistency across the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 80, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring query execution latency at multiple percentiles (p80, p95, p99, p999) across TiDB instances. Essential for identifying performance degradation, tail latency issues, and understanding query response time distribution when investigating slow query complaints or SLA violations.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572229, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log fetch", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log entry fetches in TiKV, categorized by fetch type. Helps diagnose Raft replication issues, follower lag, and log synchronization bottlenecks when replicas need to catch up with the leader.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_entry_fetches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3930, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of external storage creation operations in TiKV during backup and restore processes. Essential for diagnosing performance issues when TiKV establishes connections to external storage systems like S3 or GCS, helping identify slow storage initialization that could delay backup/restore operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 251, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Action Duration", "group": "Schema Load", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency of schema loading operations in TiDB, broken down by action type (e.g., reload, diff). Critical for diagnosing DDL performance issues and identifying slow schema synchronization that can impact query execution and cluster stability.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, action))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4936, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the P99 latency of external storage connection initialization in TiKV during backup and restore operations. Critical for diagnosing slow backup/restore performance caused by delays in establishing connections to external storage systems like S3 or GCS.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 163, "dashboard": "Prometheus-TiCDC-Clinic", "title": "The status of changefeeds", "group": "Changefeed", "used_for": "Used for monitoring the operational status of TiCDC changefeeds across the cluster. Essential for quickly identifying which changefeeds are running, stopped, or experiencing issues, enabling rapid troubleshooting of data replication problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 597, "dashboard": "Prometheus-TiCDC-Clinic", "title": "changefeed tick duration percentile", "group": "Changefeed", "used_for": "Used for monitoring the 99th and 99.9th percentile latency of TiCDC changefeed tick operations, which represent the core processing cycle for capturing and replicating data changes. Essential for diagnosing replication lag issues and identifying performance bottlenecks in CDC data synchronization pipelines.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_owner_changefeed_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))", "histogram_quantile(0.99, sum(rate(ticdc_processor_processor_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 24, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Request", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU resource requests and limits configured for each pod in the dataplane namespace. Essential for capacity planning, identifying pods with insufficient resource allocations, and detecting potential CPU throttling issues before they impact performance.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 21, "dashboard": "TiDB-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "Used for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for understanding query workload distribution, identifying traffic patterns, and detecting sudden changes in application behavior or load spikes.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Usage", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of individual pods within the selected namespace. Helps identify pods with high CPU consumption, detect resource bottlenecks, and diagnose performance issues or resource contention at the pod level.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (cluster,namespace,pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\", container!=\"POD\",container!=\"\"}[$__rate_interval])) " ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 74, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC network traffic", "group": "TiKV", "used_for": "Used for monitoring the network traffic sent by TiKV's CDC (Change Data Capture) gRPC messages, broken down by instance and message type. Essential for diagnosing CDC replication performance issues, identifying network bottlenecks, and tracking data transmission rates during change data capture operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log async fetch task duration", "group": "Raft Log", "used_for": "Used for monitoring the latency of asynchronous Raft log entry fetch tasks in TiKV at p95 and p99 percentiles. Critical for diagnosing delays in Raft log replication and identifying performance bottlenecks when followers retrieve log entries from leaders.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftlog-fetch-worker\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC CPU", "group": "TiKV", "used_for": "Used for monitoring CPU usage of TiCDC-related threads running on TiKV instances, including CDC workers, CDC event processors, and TSO threads. Essential for diagnosing performance bottlenecks in change data capture operations and identifying instances where CDC workloads are consuming excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12971, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolve address duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of address resolution operations in TiKV servers. Critical for diagnosing network-related issues, DNS resolution delays, or service discovery problems that could impact inter-node communication and overall cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_address_resolve_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 467, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write throughput and data insertion rate of RocksDB's block cache in TiKV. Essential for analyzing cache utilization patterns and diagnosing memory pressure issues when investigating storage layer performance problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 42, "dashboard": "TiDB-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (success/error). Essential for tracking overall query throughput, comparing current performance against historical baselines (1-day offset), and quickly identifying sudden drops in successful queries or spikes in failed commands that indicate cluster health issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572581, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile wait time before tasks are scheduled in TiKV's thread pools. Critical for diagnosing thread pool saturation and identifying when tasks are queuing excessively before execution, which can indicate resource contention or insufficient thread pool capacity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, name))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 137, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Physcial", "group": "Cluster", "used_for": "Used for monitoring the current physical timestamp value of PD's global TSO (Timestamp Oracle). Essential for verifying TSO progression, detecting time anomalies, and diagnosing issues where TSO may be stuck or regressing, which would cause transaction failures across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2, "dashboard": "TiDB-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring the commands per second (CPS) executed by each TiDB instance in the cluster. Helps identify uneven query distribution across instances, detect instance-level performance bottlenecks, and diagnose load balancing issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 28, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema loading operations in TiDB instances, categorized by operation type. Essential for diagnosing schema synchronization issues, identifying excessive schema reloads, and troubleshooting DDL-related performance problems across the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 5267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Backup & Import", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze requests in TiKV's coprocessor. Critical for diagnosing performance issues during backup operations, table checksums, and ANALYZE TABLE commands that verify data integrity.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 102, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD etcd requests/s", "group": "Changefeed", "used_for": "Used for monitoring the rate of etcd requests per second made by TiCDC instances to PD's etcd. Helps identify excessive etcd access patterns, potential performance bottlenecks in metadata operations, and diagnose changefeed coordination issues.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1464, "dashboard": "PD-Cluster-Clinic", "title": "Scheduling Allowance Status", "group": "Cluster", "used_for": "Used for monitoring whether PD's scheduling operations are currently allowed or restricted in the cluster. Essential for diagnosing why region scheduling might be paused or limited, helping identify if scheduling is intentionally disabled due to maintenance, resource constraints, or automatic protection mechanisms.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 157, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Data From Cached Table Duration", "group": "Schema Load", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency when loading data from TiDB's cached tables. Critical for diagnosing performance issues with table cache operations and identifying instances where cache loading becomes a bottleneck during query execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 80, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the hit rate of RocksDB's block cache in TiKV, including overall cache hits, data block hits, and filter block hits. Essential for evaluating cache effectiveness and diagnosing read performance issues—low hit rates indicate insufficient cache size or poor cache utilization, leading to increased disk I/O and slower read operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 514, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schema Storage GC progress", "group": "Changefeed", "used_for": "Used for monitoring the schema storage garbage collection progress in TiCDC changefeeds by comparing the current cluster TSO with the GC timestamp. Essential for identifying schema storage memory issues and ensuring timely cleanup of historical schema versions that are no longer needed by replication tasks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_processor_schema_storage_gc_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1443, "dashboard": "PD-Cluster-Clinic", "title": "Max TSO gap", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the maximum TSO (Timestamp Oracle) gap in milliseconds between different data centers in a PD cluster. Critical for identifying clock synchronization issues and potential data consistency problems in multi-datacenter deployments, where large TSO gaps can indicate network latency or PD leader switching issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763572692, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the average time tasks spend waiting in TiKV's thread pool queues before execution. Critical for identifying thread pool saturation and scheduling bottlenecks that can cause request processing delays and overall performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) / sum(rate(tikv_yatp_pool_schedule_wait_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1444, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Datetime", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for displaying the current timestamp oracle (TSO) value in PD, which represents the cluster's global logical time. Essential for verifying time synchronization across the cluster and diagnosing issues related to transaction ordering or time-based anomalies.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 132, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiKV's RocksDB storage engine, including keys read, keys written, and corrupt keys detected during compaction. Essential for tracking data access patterns, write throughput, and identifying potential data corruption issues that could impact cluster health.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5269, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO Utilization", "group": "Backup & Import", "used_for": "Used for monitoring disk I/O utilization percentage on TiKV nodes during backup and import operations. Critical for identifying disk bottlenecks that can cause latency spikes when performing data-intensive backup or bulk import tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Cache OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema cache operations in TiDB, broken down by action and type. Helps diagnose schema loading performance issues, cache hit/miss patterns, and identify potential bottlenecks when applications frequently access or update table metadata.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infocache_counters{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (action,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 147, "dashboard": "Prometheus-TiCDC-Clinic", "title": "gRPC message count", "group": "TiKV", "used_for": "Used for monitoring the rate of gRPC message requests handled by TiKV instances, broken down by message type. Essential for tracking TiKV's communication patterns with TiDB/TiCDC and identifying unusual spikes in specific gRPC operations that may indicate replication issues or performance bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Size", "group": "Volume", "used_for": "Used for monitoring the requested storage capacity of PersistentVolumeClaims in the specified namespace. Helps track storage resource allocation and identify potential capacity planning needs or storage quota issues in the Kubernetes cluster.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_resource_requests_storage_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572784, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO time per second", "group": "Server", "used_for": "Used for monitoring the rate of disk block read time per second in TiKV's RocksDB storage engine, broken down by request type (storage and coprocessor operations). Essential for diagnosing disk I/O bottlenecks and identifying whether slow queries are caused by storage layer read latency or coprocessor processing delays.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 267, "dashboard": "TiDB-Cluster-Clinic", "title": "Affected Rows By Type", "group": "Query Summary", "used_for": "Used for monitoring the rate of rows affected by different SQL statement types (INSERT, UPDATE, DELETE) across TiDB instances. Helps identify data modification patterns, detect unusual write activity, and diagnose workload characteristics when investigating performance issues or data changes.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type)", "sum(rate(tidb_server_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 52, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Current Usage", "group": "Volume", "used_for": "Used for monitoring the current storage consumption of PersistentVolumes in the dataplane namespace. Helps identify volumes approaching capacity limits and potential storage exhaustion issues before they impact cluster operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim) - sum(kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 194, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC memory", "group": "TiKV", "used_for": "Used for monitoring memory consumption of TiKV instances and CDC (Change Data Capture) components. Helps diagnose memory-related issues in TiKV nodes, particularly when troubleshooting CDC replication performance problems or investigating potential memory leaks in the capture process.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"tikv.*\"}) by (instance)", "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"cdc.*\"}) by (instance)", "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 468, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiKV's RocksDB storage engine, including overall blocks, data blocks, and filter blocks. Essential for understanding cache population patterns and diagnosing issues related to cache churn or excessive memory allocation in the block cache.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 137, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per minute in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing issues like connection problems, syntax errors, or resource constraints affecting query success rates.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 82, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Exit error count/m", "group": "Changefeed", "used_for": "Used for monitoring the rate of TiCDC processor exits caused by errors per minute across changefeeds. Critical for detecting changefeed failures and stability issues that require immediate attention to prevent data replication interruptions.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance, namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5925, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import or restore operations. Essential for diagnosing performance bottlenecks during backup restoration, Lightning imports, or BR (Backup & Restore) operations by tracking per-instance CPU consumption and thread activity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572785, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO bytes per second", "group": "Server", "used_for": "Used for monitoring the disk I/O throughput in bytes per second for RocksDB read operations across TiKV instances. Essential for diagnosing storage performance bottlenecks, identifying excessive disk read activity, and correlating I/O patterns with query performance issues in both storage and coprocessor requests.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2292, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Local reader requests", "group": "Local reader", "used_for": "Used for monitoring TiKV's local reader request patterns, including successfully executed requests, stale read requests, and rejected requests with their reasons. Essential for diagnosing local read optimization effectiveness and identifying why read requests might be falling back to the Raft consensus path instead of being served locally.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_local_read_reject_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", "sum(rate(tikv_raftstore_local_read_executed_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572002, "dashboard": "TiDB-Cluster-Clinic", "title": "Lease Duration", "group": "Schema Load", "used_for": "Used for monitoring the remaining time until TiDB's schema lease expires. Critical for diagnosing schema synchronization issues and detecting potential schema lease expiration problems that could cause DDL operations to fail or stall.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} - time()" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 5926, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of active SST (Sorted String Table) import threads in TiKV instances during data import or restore operations. Essential for diagnosing import performance bottlenecks and verifying that backup/restore jobs are actively processing data across the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 85, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of RocksDB storage engine in TiKV, tracking both direct bytes read and iterator-based reads. Essential for diagnosing read I/O bottlenecks, understanding read workload patterns, and identifying potential performance issues in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 45, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator create", "group": "Operator", "used_for": "Used for monitoring the rate at which PD creates different types of scheduling operators. Essential for understanding cluster rebalancing activity and diagnosing whether PD is actively responding to region distribution imbalances or failures.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 153, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved Region", "group": "TiKV", "used_for": "Used for monitoring the minimum resolved timestamp region across TiKV instances in TiCDC replication. Helps identify lagging regions that may be blocking CDC replication progress and causing data synchronization delays.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 218, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when event processing slows down or stalls.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 112, "dashboard": "TiDB-Cluster-Clinic", "title": "Slow query", "group": "Query Summary", "used_for": "Used for monitoring the 90th percentile latency breakdown of slow queries in TiDB, including total processing time, coprocessor execution time, and wait time. Essential for diagnosing performance bottlenecks in slow queries and identifying whether delays occur during query processing, storage layer operations, or resource waiting.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 22, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolumeClaim", "group": "Volume", "used_for": "Used for displaying information about PersistentVolumeClaims (PVCs) in the specified Kubernetes namespace. Helps administrators track storage volume claims and verify that pods have the necessary persistent storage resources allocated for data persistence.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4194, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiKV's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate read workload imbalances or priority queue bottlenecks affecting query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Duration 95", "group": "DDL", "used_for": "Used for monitoring the 95th percentile latency of DDL (Data Definition Language) operations in TiDB, including schema changes and index creation. Critical for diagnosing slow DDL performance issues and identifying operations that may be blocking schema modifications or causing application timeouts.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 131, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys stored in each RocksDB column family within TiKV. Helps assess data distribution across column families and identify potential storage imbalances or unexpected data growth patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 152, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved ts", "group": "TiKV", "used_for": "Used for monitoring the replication lag between TiKV's minimum resolved timestamp and the current cluster TSO in TiCDC. Critical for identifying data replication delays and ensuring change data capture is keeping up with real-time transactions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5932, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Errors", "group": "Backup & Import", "used_for": "Used for monitoring the rate of errors occurring during data import operations in TiKV. Essential for quickly identifying import failures or issues when restoring backups or bulk loading data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 95, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of different gRPC message types processed by TiKV instances. Helps identify which gRPC operations (like Coprocessor requests, prewrite, commit) are most active and diagnose communication bottlenecks between TiDB/TiKV components.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4196, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of read operations hitting Level 0 SST files in TiKV's unified read pool. High Level 0 chance indicates potential read amplification and compaction pressure, which can degrade read performance and signal the need for tuning RocksDB compaction settings.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 136, "dashboard": "TiDB-Cluster-Clinic", "title": "999 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile (P999) query execution duration in TiDB, broken down by SQL type. Critical for identifying tail latency issues and detecting slow query outliers that affect the worst-case user experience, helping diagnose performance degradation before it impacts the majority of queries.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 229, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events", "group": "Dataflow", "used_for": "Used for monitoring the number of events collected and output by TiCDC's puller component across changefeeds. Essential for tracking data replication throughput and diagnosing data flow bottlenecks or stalls in the CDC pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 70, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Resolved ts lag duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.999th percentile lag between TiKV's resolved timestamp and current time in TiCDC replication. Critical for identifying extreme tail latency in change data capture, helping diagnose replication delays and ensuring data consistency in downstream systems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 68, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV takes to perform initial table scans when TiCDC starts replicating data. Critical for diagnosing slow changefeed initialization and identifying performance bottlenecks during the initial data snapshot phase.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 134, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution latency in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation and diagnosing slow query issues affecting the majority of database operations.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4198, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiKV's unified read pool across instances. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient thread pool capacity or slow read operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "Dataplane Resource Overview", "title": "Docker Images", "group": "Image Versions", "used_for": "Used for monitoring the container images deployed across pods in the dataplane namespace. Helps track image versions, identify version inconsistencies, and verify successful deployments or rollouts across the cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_pod_container_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 63, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Backfill Data Rate", "group": "DDL", "used_for": "Used for monitoring the rate at which DDL operations backfill data during index creation in TiDB. Essential for tracking DDL job progress and diagnosing slow index building performance, helping identify bottlenecks when adding indexes to large tables.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_add_index_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Backup & Import", "used_for": "Used for monitoring the rate of import RPC operations per second in TiKV, broken down by instance and request type. Essential for tracking data import activity and diagnosing performance issues during backup restoration or bulk data loading operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (instance, request)", "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (request)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 218, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Idle Duration", "group": "Query Summary", "used_for": "Used for monitoring the idle time duration of database connections in TiDB, separated by whether connections are in a transaction (in_txn) or not. Essential for identifying connection pool inefficiencies, detecting long-running idle transactions that may hold locks, and diagnosing resource waste from improperly managed connections.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 107, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV, broken down by message type. Critical for identifying communication failures between TiKV and other components (TiDB, PD, or other TiKV nodes), helping diagnose network issues, timeout problems, or service unavailability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 228, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component per second. Essential for tracking data flow throughput and identifying bottlenecks in the change data capture pipeline, particularly when diagnosing replication lag or sorter performance issues.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 86, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the write throughput in TiKV's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written per second. Essential for diagnosing write performance bottlenecks, identifying I/O saturation issues, and understanding storage layer write patterns during high-load scenarios.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 220, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component across changefeeds. Essential for diagnosing data replication throughput issues and identifying bottlenecks in the event processing pipeline when replication lag occurs.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, instance, changefeed, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 62, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Waiting Jobs Count", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs waiting in the queue across TiDB instances. Critical for identifying DDL operation bottlenecks, schema change delays, and potential blocking issues that could impact database schema modifications.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572469, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Unified Read Pool Wait Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the wait time distribution of tasks in TiKV's unified read pool before execution begins. Critical for diagnosing read request queuing issues and identifying when the read pool is saturated, causing increased query latency due to thread pool contention.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified-read.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator timeout", "group": "Operator", "used_for": "Used for monitoring the number of PD scheduling operators that have timed out within a 2-minute window, categorized by operator type. Critical for identifying scheduling bottlenecks and diagnosing why region balancing or leader transfers are failing to complete in the expected timeframe.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"timeout\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 98, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message processing in TiKV, excluding garbage collection operations. Critical for identifying performance bottlenecks in inter-component communication and diagnosing slow gRPC requests that may impact overall cluster responsiveness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))", "histogram_quantile(0.99, tikv_grpc_msg_duration_seconds_bucket:le_type:sum:rate{type!=\"kv_gc\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.99th percentile latency of TiCDC's initial table scan operations in TiKV. Critical for diagnosing slow initial replication setup and identifying outlier scan performance issues that could delay CDC task initialization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 77, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator finish", "group": "Operator", "used_for": "Used for monitoring the completion rate of different types of scheduling operators in PD over 2-minute intervals. Helps track scheduling activity effectiveness and identify which operator types (like region balance, leader transfer, or replica management) are actively completing their tasks, useful for diagnosing scheduling performance and cluster rebalancing progress.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"finish\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5931, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the 99th percentile latency of import RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets through tools like TiDB Lightning or BR (Backup & Restore), helping identify slow import requests that may impact overall import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))", "histogram_quantile(0.5, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data read per RocksDB read operation in TiKV, showing maximum, 99th, and 95th percentile values. Helps diagnose read amplification issues and identify whether queries are reading excessive amounts of data, which can indicate inefficient data access patterns or compaction problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 90, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction I/O throughput in TiKV, tracking bytes read during compaction, bytes written after compaction, and flush write operations. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification that may impact overall cluster I/O efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 132, "dashboard": "TiDB-Cluster-Clinic", "title": "95 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation affecting most queries and establishing baseline latency expectations for different workload patterns.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 219, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's mounter component per second. Essential for tracking data change capture throughput and identifying bottlenecks in the CDC pipeline where row-level changes are being processed and transformed.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5930, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, helping identify whether slow imports are caused by write or download operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator replaced or canceled", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that are canceled or replaced, broken down by operator type. Helps diagnose scheduling instability, conflicting operations, or frequent strategy changes that may indicate cluster rebalancing issues or configuration problems.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"cancel\"}[2m])) by (type)", "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"replace\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 67, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb)", "group": "", "used_for": "Used for monitoring Horizontal Pod Autoscaler (HPA) status for TiDB instances, showing desired, current, and maximum replica counts. Essential for tracking auto-scaling behavior and ensuring TiDB pods scale appropriately based on workload demands.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 140, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan tasks status", "group": "TiKV", "used_for": "Used for monitoring the status of TiCDC initial scan tasks in TiKV, showing both ongoing and completed scans per instance. Essential for tracking TiCDC replication initialization progress and identifying instances with stalled or slow initial data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2532, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of different gRPC message types in TiKV. Helps identify which specific gRPC operations (like coprocessor requests, prewrite, commit) are experiencing performance degradation or contributing to overall cluster latency issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 134, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiKV's RocksDB storage engine, showing maximum, P99, and P95 values. Helps diagnose write amplification issues and identify whether write operations are batching efficiently or writing unexpectedly large amounts of data per operation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Worker Duration 99", "group": "DDL", "used_for": "Used for monitoring the 99th percentile latency of DDL worker operations in TiDB, broken down by operation type and action. Critical for diagnosing slow schema changes and identifying DDL performance bottlenecks that may impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, action, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4199, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Duration of One Time Slice", "group": "Unified Read Pool", "used_for": "**Used For:**\n\nUsed for monitoring the duration of individual task execution time slices in TiKV's unified read pool at various percentiles (p50, p95, p99, p999). Essential for diagnosing read request processing efficiency and identifying whether tasks are being preempted too frequently or holding execution threads too long, which can impact overall read performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator check", "group": "Operator", "used_for": "Used for monitoring the rate of operator checks performed by PD's scheduler across different operator types. Helps diagnose scheduling activity levels and identify which types of operators are being evaluated most frequently, useful for understanding PD's scheduling behavior and potential bottlenecks.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"check\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 6905, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of saving SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup data persistence process.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operators count by state", "group": "Operator", "used_for": "**Used For:**\n\nUsed for monitoring the rate of change in PD scheduler operator counts grouped by event state (created, finished, timeout, canceled, replaced). Essential for diagnosing scheduling behavior issues, identifying stuck operators, and understanding whether the cluster is actively rebalancing regions or experiencing scheduling bottlenecks.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 130, "dashboard": "TiDB-Cluster-Clinic", "title": "80 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution latency in TiDB, showing the duration threshold that 80% of queries complete within. Helps identify typical query performance and detect degradation affecting the majority of queries, complementing p99/p999 metrics for a complete latency profile.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4202, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Execute Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the execution latency of tasks in TiKV's unified read pool across different percentiles (p50, p95, p99, p999). Essential for diagnosing read performance issues and identifying whether the unified read pool is experiencing task execution delays that could impact query response times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2533, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the batch size of gRPC requests and responses in TiKV, showing both p99 percentile and average values. Helps diagnose gRPC batching efficiency and identify potential network communication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 64, "dashboard": "TiDB-Cluster-Clinic", "title": "Deploy Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the maximum latency of DDL syncer deployment operations in TiDB. Critical for diagnosing delays in DDL synchronization across the cluster, particularly when schema changes are slow to propagate or fail to deploy.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6906, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Read SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of reading SST (Sorted String Table) files during data import operations in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the data ingestion pipeline, particularly when restoring backups or migrating data.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 518, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiKV's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O and degrade read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 67, "dashboard": "PD-Cluster-Clinic", "title": "Operator finish duration", "group": "Operator", "used_for": "Used for monitoring the p95 and p99 latency of PD operator completion times by operator type. Essential for diagnosing scheduling performance issues and identifying which operator types (like region merge, leader transfer, or replica addition) are experiencing delays in execution.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5929, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Wait Duration", "group": "Backup & Import", "used_for": "Used for monitoring the wait time in the import queue when TiKV is downloading data during backup/restore operations. Helps identify bottlenecks in the import pipeline when data is queued waiting to be processed, indicating potential resource constraints or slow download speeds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Captured region count", "group": "TiKV", "used_for": "Used for monitoring the number of regions captured by TiCDC on each TiKV instance and their resolution status. Essential for diagnosing TiCDC replication issues, identifying regions that are stuck or slow to replicate, and ensuring change data capture is working correctly across all regions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 224, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of row change events output by TiCDC's mounter component, which parses TiKV change data into structured events. Essential for tracking data replication throughput and diagnosing pipeline bottlenecks when changefeed lag increases or replication stalls occur.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2534, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages sent between TiKV nodes via gRPC. Helps diagnose Raft communication efficiency and identify potential network batching issues that could impact cluster consensus performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 80 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution duration across individual TiDB instances. Helps identify performance variations between instances and pinpoint which specific TiDB nodes are experiencing slower query processing times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 127, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the total bytes of data waiting to be compacted in RocksDB column families. Critical for identifying compaction backlogs that can lead to read amplification, write stalls, and degraded I/O performance in TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 76, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC scan speed", "group": "TiKV", "used_for": "Used for monitoring the rate at which TiKV scans data for CDC (Change Data Capture) replication, measured in bytes per second per instance. Essential for diagnosing CDC performance bottlenecks, identifying slow TiKV nodes during initial table scans, and ensuring efficient data synchronization to downstream systems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4204, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Schedule Times", "group": "Unified Read Pool", "used_for": "**Used For:**\n\nUsed for monitoring the execution time distribution of tasks in TiKV's unified read pool at various percentiles (p50, p95, p99, p999). Essential for diagnosing read performance issues and identifying whether read operations are experiencing scheduling delays or execution bottlenecks in the thread pool.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring TiDB pod CPU utilization in relation to Horizontal Pod Autoscaler (HPA) targets. Helps operators verify if HPA scaling thresholds are properly configured and whether actual CPU usage aligns with target metrics, enabling proactive capacity planning and autoscaling troubleshooting.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n", "(\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n /\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n) * 100" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 65, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Handle Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the maximum latency of DDL owner operations when handling schema synchronization across TiDB nodes. Critical for diagnosing DDL execution delays and identifying issues with schema change propagation in distributed environments.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 55, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL META OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per minute in TiDB, tracking the rate of schema changes like CREATE, ALTER, and DROP statements. Essential for diagnosing DDL performance issues, identifying schema change bottlenecks, and understanding the frequency of metadata operations across TiDB instances.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 223, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of data change events being written to downstream tables by TiCDC's table sink component. Essential for tracking replication throughput and identifying bottlenecks in the data synchronization pipeline when events are not being delivered at expected rates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 58, "dashboard": "Dataplane Resource Overview", "title": "Avg Mem Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring average memory utilization of TiDB pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether memory-based autoscaling is functioning correctly or if pods are consistently over/under-provisioned.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "1", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"} and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 66, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Self Version Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL owner updating its version information in the cluster. Critical for diagnosing DDL coordination issues and detecting delays in schema change propagation across TiDB instances.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572858, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources QPS", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC requests to TiKV broken down by source (e.g., TiDB, PD, other TiKV nodes). Helps identify which components are generating the most traffic to TiKV and diagnose imbalanced request patterns or unexpected traffic sources.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_request_source_counter_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "PD-Cluster-Clinic", "title": "Operator step finish duration", "group": "Operator", "used_for": "Used for monitoring the p95 and p99 latency of PD operator step completion during cluster scheduling operations. Essential for diagnosing scheduling performance issues and identifying bottlenecks in region migration, leader transfer, and other operator execution steps.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5928, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data or migrating large datasets, as slow rewrite operations can significantly impact overall import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 143, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache hit", "group": "TiKV", "used_for": "Used for monitoring the cache hit rate of TiCDC's old value cache in TiKV instances. Essential for evaluating TiCDC replication efficiency and identifying memory pressure issues that could impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 516, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiKV instances. Helps identify potential memory leaks or snapshot accumulation issues that can impact performance and resource usage when snapshots are not properly released.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC total scan bytes", "group": "TiKV", "used_for": "Used for monitoring the total bytes scanned by TiCDC (Change Data Capture) on each TiKV instance. Helps identify data replication workload intensity and diagnose performance issues when CDC is reading historical data or catching up on changes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5939, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of import ingest RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when ingesting SST files during backup restoration or bulk data import operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 664, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of rows being flushed to downstream sinks per second in TiCDC. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when rows aren't being written to downstream systems as expected.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 221, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of data change events output by TiCDC's table sink to downstream systems. Essential for tracking replication throughput, verifying data synchronization progress, and diagnosing potential bottlenecks in the CDC pipeline when event output rates are lower than expected.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 68, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiDB worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for verifying autoscaling behavior and identifying when worker pods are scaling up/down in response to workload changes or hitting configured capacity limits.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 95 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration for each TiDB instance. Helps identify performance variations across individual TiDB nodes and pinpoint specific instances experiencing elevated query latency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 25, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution duration for each TiDB instance. Helps identify performance bottlenecks and uneven load distribution across individual TiDB servers by comparing p99 latencies between instances.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 863, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression efficiency of data stored in TiKV's RocksDB engine across different column families and LSM tree levels. Essential for evaluating storage space utilization and identifying compression issues that could lead to excessive disk usage or degraded I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1426, "dashboard": "PD-Cluster-Clinic", "title": "Schedulers reach limit", "group": "Operator", "used_for": "Used for monitoring when PD schedulers hit their configured limits and cannot create new operators. Critical for diagnosing why region scheduling stops or slows down, helping identify if operator limits or per-store limits are constraining cluster rebalancing and data migration.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,name)", "sum(delta(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (desc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 63, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tidb_worker_hint(tidb-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiDB worker pods, comparing target scaling thresholds against current metric values. Helps operators understand when and why TiDB workers are being scaled up or down based on the custom `dxf_hscale_tidb_worker_hint` metric in Kubernetes environments.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"} *1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763573090, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC resource group QPS", "group": "gRPC", "used_for": "Used for monitoring the queries per second (QPS) for each resource group in TiKV's gRPC layer. Helps identify resource group usage patterns, detect imbalanced workload distribution across groups, and diagnose performance issues related to resource control policies.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_resource_group_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring the rate of different storage commands (Get, Scan, Prewrite, Commit, etc.) processed by TiKV instances. Essential for understanding workload patterns, identifying command-specific bottlenecks, and diagnosing transaction processing issues in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 190, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per minute in TiDB, broken down by operation type (e.g., CREATE, ALTER, DROP). Essential for tracking schema change frequency and identifying periods of heavy DDL activity that may impact cluster performance or cause lock contention.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572859, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources duration", "group": "gRPC", "used_for": "Used for monitoring the duration of gRPC requests grouped by their source in TiKV. Helps identify which client components (TiDB, PD, or other TiKV nodes) are generating the most gRPC traffic and diagnose communication bottlenecks between cluster components.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_request_source_duration_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5937, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "Backup & Import", "used_for": "Used for monitoring the rate of SST (Sorted String Table) file ingestion during data import operations in TiKV. Essential for tracking import throughput and diagnosing performance bottlenecks when restoring backups or bulk loading data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1455, "dashboard": "PD-Cluster-Clinic", "title": "99% operator region size", "group": "Operator", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile size of regions being processed by PD operators. Helps identify whether operators are handling unusually large regions, which can indicate region merge/split issues or unbalanced data distribution affecting scheduling performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type,le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 192, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL backfill progress in percentage", "group": "DDL", "used_for": "Used for tracking the completion percentage of DDL backfill operations in TiDB, including index creation and column modifications. Essential for monitoring long-running schema changes and estimating when DDL operations will complete.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"modify_column\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 70, "dashboard": "Dataplane Resource Overview", "title": "HPA info(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of coprocessor worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for understanding autoscaling behavior and identifying whether coprocessor workers are scaling appropriately to handle workload demands or hitting configured capacity limits.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 665, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows", "group": "Dataflow", "used_for": "Used for monitoring the total number of rows flushed to the downstream sink by TiCDC changefeeds. Essential for tracking data replication throughput and identifying changefeeds with abnormal or stalled data flow to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2002, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in RocksDB's LSM-tree structure for each column family. Essential for diagnosing compaction issues, identifying level imbalances, and understanding storage engine health that can impact read/write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 999 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile query execution duration for each individual TiDB instance. Essential for identifying specific TiDB nodes experiencing tail latency issues or performance degradation, enabling targeted troubleshooting of slow query problems at the instance level.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiKV, broken down by error status type. Critical for identifying storage engine issues, detecting data access failures, and diagnosing underlying problems with RocksDB operations or disk I/O errors.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5938, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, as high ingestion duration directly impacts import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 145, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache size", "group": "TiKV", "used_for": "Used for monitoring TiCDC's old value cache in TiKV, tracking the number of cached entries, total memory usage, and memory quota per instance. Critical for diagnosing TiCDC replication issues and ensuring the old value cache doesn't exceed memory limits, which could impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 141, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek operation", "group": "TiKV", "used_for": "Used for monitoring the rate of old value scan operations in TiKV's Change Data Capture (CDC) feature, broken down by instance, column family, and tag. Essential for diagnosing CDC performance issues and understanding the I/O overhead of scanning historical data when capturing change events.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, cf, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 15, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous snapshot requests in TiKV's storage engine. Critical for diagnosing read performance issues and identifying delays in snapshot acquisition that could impact transaction consistency checks and point-in-time reads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 83, "dashboard": "PD-Cluster-Clinic", "title": "Store capacity", "group": "Statistics - balance", "used_for": "Used for monitoring the total storage capacity of each TiKV store in the cluster. Essential for capacity planning and identifying stores approaching their storage limits before they impact cluster availability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_capacity\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 109, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying when async write operations are experiencing delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of coprocessor-worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Helps determine if HPA scaling policies are appropriately configured and whether coprocessor workers need scaling adjustments to handle analytical query workloads efficiently.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 1.1", "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 0.9", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-coprocessor)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5927, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during data import operations in TiKV. Helps diagnose slow import performance by identifying network bandwidth bottlenecks or download speed issues when restoring backups or importing large datasets.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 517, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiKV's RocksDB storage engine. Critical for identifying long-running transactions or stale snapshots that can block compaction, cause disk space bloat, and degrade overall storage performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 719, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Quota", "group": "Sink - General", "used_for": "Used for monitoring memory quota allocation and redo event cache usage in TiCDC's sink manager. Critical for diagnosing memory pressure issues, preventing OOM errors, and ensuring the changefeed has sufficient memory resources to buffer events during replication.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sinkmanager_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", component!=\"\", instance=~\"$ticdc_instance\"}", "ticdc_sinkmanager_redo_event_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 146, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV spends seeking old values during CDC (Change Data Capture) operations. Critical for diagnosing CDC replication performance issues and identifying bottlenecks in reading historical data versions required for change feeds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 24763573092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages received by each TiKV store instance. Helps identify communication patterns between TiKV nodes and diagnose potential network issues or imbalanced message distribution in the Raft consensus layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_message_recv_by_store{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM Detail", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the per-minute increase in query execution errors across TiDB instances. Essential for identifying which specific TiDB servers are experiencing query failures and diagnosing application-level errors, connection issues, or SQL syntax problems.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 264, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL job table duration", "group": "DDL", "used_for": "Used for monitoring the 95th percentile duration of DDL job table operations in TiDB. Critical for diagnosing slow DDL execution and identifying performance bottlenecks in schema changes, helping pinpoint whether delays occur during job table reads/writes or metadata operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", }[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 75, "dashboard": "Dataplane Resource Overview", "title": "Scaling Event Distribution(coprocessor-worker)", "group": "", "used_for": "Used for tracking horizontal pod autoscaler (HPA) scaling events for coprocessor workers in the TiDB dataplane. Helps identify when and how frequently the coprocessor worker pool scales up or down in response to workload changes, essential for validating autoscaling behavior and capacity planning.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace" ], "promql": [ "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} > bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])", "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} < bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 91, "dashboard": "PD-Cluster-Clinic", "title": "Store available", "group": "Statistics - balance", "used_for": "Used for monitoring the number of available stores in the TiKV cluster from PD's perspective. Essential for verifying cluster capacity and diagnosing store availability issues that could impact data placement and load balancing decisions.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 265, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Running Job Cnt", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs currently executing in TiDB. Helps identify DDL operation bottlenecks, track schema change progress, and diagnose issues when DDL operations appear stuck or accumulate unexpectedly.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_running_job_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12309, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write keys", "group": "Backup & Import", "used_for": "Used for monitoring the number of keys written during local import operations in TiKV over 2-minute intervals. Helps track import progress and diagnose slow data ingestion issues when using TiDB Lightning or BR (Backup & Restore) local backend mode.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_local_write_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2381, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for tracking the frequency of RocksDB stall condition changes in TiKV's storage engine. Essential for diagnosing write throttling and I/O pressure issues, as frequent stall condition changes indicate the storage engine is struggling with write amplification or compaction backlog.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20000, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine at p95 and p99 percentiles. Critical for diagnosing read performance issues and identifying delays in obtaining consistent snapshots for transactions and queries.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "TiDB-Cluster-Clinic", "title": "Internal SQL OPS", "group": "Query Detail", "used_for": "Used for monitoring the throughput of internal SQL operations executed by TiDB, such as statistics collection, auto-analyze, and system maintenance queries. Essential for distinguishing internal database overhead from user workload and diagnosing performance issues caused by background tasks.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 721, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Event Cache Hit Ratio", "group": "Sink - General", "used_for": "Used for monitoring the cache hit ratio of redo log events in TiCDC's sink manager. Helps diagnose redo log performance issues by showing how effectively events are being served from cache versus requiring disk I/O, with low ratios indicating potential memory pressure or inefficient caching.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", type=\"hit\"}[2m])) /\nsum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 142, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the latency of TiKV seeking old values during TiCDC change data capture operations, showing p95 and p99 percentiles. Critical for diagnosing TiCDC replication performance issues and identifying instances where historical value lookups are causing delays in change feed processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 90, "dashboard": "PD-Cluster-Clinic", "title": "Store used", "group": "Statistics - balance", "used_for": "Used for monitoring the actual disk space consumed by each TiKV store in the cluster. Essential for tracking storage utilization trends, identifying stores approaching capacity limits, and diagnosing uneven data distribution issues that may require rebalancing.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_used\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 61, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's Raft store threads, which handle Raft consensus operations including log replication and region management. Essential for diagnosing performance bottlenecks related to Raft consensus overhead, identifying instances with high replication load, and troubleshooting slow write operations caused by Raft processing delays.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "Dataplane Resource Overview", "title": "Scaling Evnt Detials(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaling behavior of coprocessor workers, tracking current replicas, desired replicas, and the scaling gap. Essential for diagnosing autoscaling responsiveness issues and determining whether coprocessor resources are scaling appropriately to meet workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "last_over_time(kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])", "last_over_time(kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])\n", "(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n -\n kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n) != 0", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval])\n", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset -15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset 15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval])", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset -15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset 15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval])\n", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset -15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset 15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"}[$__interval:$__interval])\n/\nlast_over_time(kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}[$__interval:$__interval])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 272, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries in Multi-Statement", "group": "Query Detail", "used_for": "Used for monitoring the average number of queries contained within multi-statement requests and the total rate of queries in multi-statements. Helps identify applications using batch query patterns and diagnose potential performance issues related to multi-statement execution overhead.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/sum(rate(tidb_server_multi_query_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 612, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Count (per second)", "group": "Sink - General", "used_for": "Used for monitoring the rate of rows being written to downstream sinks per second by TiCDC changefeeds. Essential for tracking replication throughput and identifying bottlenecks or drops in data synchronization performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572001, "dashboard": "TiDB-Cluster-Clinic", "title": "Add Index Scan Rate", "group": "DDL", "used_for": "Used for monitoring the row scanning throughput (rows per second) during ADD INDEX operations in TiDB at different percentiles (p50, p95, p999). Essential for evaluating DDL performance and diagnosing slow index creation issues by identifying whether the scanning phase is the bottleneck.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2003, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST (Sorted String Table) file ingestion operations in TiKV's RocksDB engine. Critical for diagnosing performance issues during snapshot recovery, data import operations, or when TiKV ingests external SST files, as high latency indicates I/O bottlenecks or storage layer problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12310, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write bytes", "group": "Backup & Import", "used_for": "Used for monitoring the rate of data bytes written during TiDB Lightning's local import operations in TiKV. Helps diagnose import performance issues and track data ingestion throughput when using the local backend import mode.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_local_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 321, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Receive Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages received by TiCDC instances from peer nodes. Helps identify communication bottlenecks, network issues, or replication lag by tracking message throughput between TiCDC servers.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 79, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's async apply threads, which handle applying Raft log entries to the RocksDB storage engine. Essential for diagnosing write performance bottlenecks and identifying instances where apply threads are saturated, which can cause write stalls and increased latency.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 84, "dashboard": "PD-Cluster-Clinic", "title": "Store available ratio", "group": "Statistics - balance", "used_for": "Used for monitoring the ratio of available storage capacity to total capacity for each TiKV store. Critical for identifying stores approaching capacity limits and preventing out-of-space issues that could impact cluster availability and data balance.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_capacity\"}) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23763572861, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL Expired", "group": "Backup & Import", "used_for": "Used for monitoring the count of expired TTL (Time-To-Live) entries during backup operations in TiKV. Helps identify data retention issues and diagnose backup performance problems caused by processing large volumes of expired raw key-value pairs.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31111, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous snapshot operations that perform pure local reads in TiKV's storage engine. Critical for diagnosing performance issues with local read operations and identifying when snapshot acquisition is causing delays in query execution.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 184, "dashboard": "TiDB-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Essential for tracking server stability, identifying recent restarts or crashes, and correlating service disruptions with specific instance downtime events.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 79, "dashboard": "Dataplane Resource Overview", "title": "HPA Evaluation Result(coprocessor-worker)", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the Horizontal Pod Autoscaler (HPA) evaluation results for coprocessor worker pods in the dataplane. Helps determine if autoscaling decisions are being made correctly based on resource utilization, enabling operators to validate scaling behavior and troubleshoot issues where pods aren't scaling as expected during coprocessor workload changes.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 13115, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store writer CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's store writer threads, which handle Raft log writes and apply operations to RocksDB. Essential for diagnosing write bottlenecks and identifying instances where store writer threads are saturated, causing write performance degradation.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_write.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tikv-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiKV worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for tracking auto-scaling behavior and identifying when TiKV workers are scaling up/down in response to load changes or hitting configured capacity limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 322, "dashboard": "TiDB-Cluster-Clinic", "title": "Task Status", "group": "Dist Execute Framework", "used_for": "Used for monitoring the status distribution of distributed tasks in TiDB's distributed execution framework. Helps identify task execution states (pending, running, completed, failed) and diagnose issues with distributed operations like DDL, import, or analytics workloads.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2452, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for identifying the root causes of write stalls in TiKV's RocksDB storage engine by tracking different stall reason types. Essential for diagnosing write performance degradation and understanding whether stalls are caused by memtable limits, L0 file accumulation, pending compaction bytes, or other RocksDB flow control mechanisms.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 610, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Batch Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile and average number of rows batched together in TiCDC sink output operations. Critical for diagnosing sink throughput efficiency and identifying whether batch sizes are optimal or if changefeeds are experiencing batching issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sinkv2_batch_row_count_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31112, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the time spent waiting for read index proposals in TiKV's storage engine. Critical for diagnosing read latency issues caused by Raft consensus delays, particularly when follower reads experience slow proposal acknowledgments from the leader.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 87, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiKV's RocksDB storage engine, tracking maximum, P99, and P95 latencies when writes are temporarily blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when RocksDB flow control mechanisms are throttling write operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31113, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's storage engine. Critical for diagnosing read performance issues related to Raft read index protocol, especially when investigating slow queries that require strong consistency guarantees.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC threads in TiKV instances, which handle client requests and inter-node communication. Essential for identifying when gRPC thread pools are CPU-saturated, causing request processing delays or connection handling bottlenecks.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 41, "dashboard": "PD-Cluster-Clinic", "title": "Store Region score", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the region distribution score across TiKV stores to evaluate load balance quality. Essential for diagnosing uneven region distribution issues and tracking the effectiveness of PD's balance-region-scheduler in redistributing regions across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_score\"}", "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}", "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 4464, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiKV's encryption-at-rest system. Helps verify encryption key management health and diagnose issues with data encryption key rotation or storage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 85, "dashboard": "PD-Cluster-Clinic", "title": "Size amplification", "group": "Statistics - balance", "used_for": "Used for monitoring the storage amplification ratio in TiKV stores by comparing region size to available disk space. Helps identify stores with inefficient space utilization, excessive data fragmentation, or compaction issues that may lead to premature disk space exhaustion.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_used\"}) by (address, store) * 2^20" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 60, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tikv-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of TiKV worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether CPU-based autoscaling is functioning correctly to handle workload changes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\", \n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-default)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 352, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Batch Size", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message batch sizes in TiCDC's peer-to-peer communication at p50, p95, and p99 percentiles. Helps diagnose message batching efficiency and identify potential network or replication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 323, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Send Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages sent by TiCDC instances to downstream systems or other peers. Helps identify message throughput bottlenecks, replication lag issues, and abnormal communication patterns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "TiDB-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiDB server memory consumption, including process resident memory and Go heap allocation. Essential for diagnosing memory leaks, OOM risks, and optimizing memory configuration when TiDB instances experience high memory pressure or crashes.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_gc_heap_goal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}) by (module, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 64, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's scheduler worker threads, which handle transaction scheduling and conflict detection. Essential for diagnosing transaction processing bottlenecks and identifying instances with high scheduling overhead that may impact write performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 103, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables across different column families in TiKV. Essential for diagnosing memory pressure issues and understanding write buffer usage, as excessive memtable size can indicate write stalls or memory configuration problems.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables-all\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tikv_worker_hint(tikv-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiKV worker pods, comparing target thresholds against current metric values. Helps operators understand autoscaling behavior and diagnose whether TiKV workers are scaling appropriately based on the custom `dxf_hscale_tikv_worker_hint_v2` metric.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 20001, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the p95 and p99 latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying when storage layer write requests experience delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 168, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring CPU utilization of TiDB server instances compared to their maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiDB instances need vertical scaling or load redistribution.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 326, "dashboard": "TiDB-Cluster-Clinic", "title": "Dist Task Subtasks", "group": "Dist Execute Framework", "used_for": "Used for monitoring the status and count of distributed task subtasks in TiDB's distributed execution framework. Helps track task completion rates and identify failed or stuck subtasks by comparing completed subtasks (succeed/failed/canceled/reverted) against total subtasks across different task types.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"succeed|failed|canceled|reverted|revert_failed\"}) by (task_id, task_type)", "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (task_id, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12712, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingestion picked level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of LSM tree levels where ingested SST files are placed in TiKV's RocksDB engine. Helps diagnose ingestion efficiency and potential write amplification issues by showing whether files are being placed at optimal levels during bulk data imports or snapshot operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_engine_ingestion_picked_level_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 611, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Large Row (size >= 128K) Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile size of large rows (≥128KB) being replicated by TiCDC to downstream sinks. Critical for identifying oversized row issues that can cause replication bottlenecks, memory pressure, or downstream write failures in change data capture pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_large_row_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_large_row_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sink_large_row_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4554, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiKV's storage layer. Helps verify encryption-at-rest is functioning correctly and track the volume of encrypted data files across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 73, "dashboard": "Dataplane Resource Overview", "title": "Scale Out summay[WIP](coprocessor-worker)", "group": "", "used_for": "Used for monitoring the autoscaling behavior of TiDB coprocessor worker pods in Kubernetes environments. Tracks CPU utilization thresholds, scaling events, and desired replica changes to help diagnose whether horizontal pod autoscaling is responding appropriately to workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", " kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1", " delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n <=\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) <= 0\n )\n )[$__range:15s]\n))" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1908, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether storage read threads are saturated or unevenly loaded across instances.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether TiKV's encryption-at-rest feature has been successfully initialized. Essential for confirming encryption status during cluster startup or when troubleshooting encryption configuration issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 614, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink Error Count (1min)", "group": "Sink - General", "used_for": "Used for tracking the rate of sink execution errors in TiCDC changefeeds over a 1-minute window. Critical for identifying downstream replication failures, connection issues, or data format problems that prevent CDC events from being successfully written to the target system.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763571993, "dashboard": "TiDB-Cluster-Clinic", "title": "Distributed Task Running Subtask Distribution on TiDB Nodes", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of active subtasks across TiDB nodes in the distributed execution framework. Helps identify workload imbalance and diagnose whether subtasks are evenly distributed among TiDB instances, which is critical for optimizing parallel execution performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending|running|reverting|revert_pending|paused\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 31114, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of async snapshot operations that are served entirely from local TiKV storage without requiring network reads from other replicas. Critical for diagnosing read performance issues in scenarios where data is available locally, helping distinguish between local storage bottlenecks and distributed read coordination overhead.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 57, "dashboard": "PD-Cluster-Clinic", "title": "Store Region size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of regions stored on each TiKV store in the cluster. Essential for identifying storage imbalance issues and verifying that PD's region scheduling is distributing data evenly across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 40, "dashboard": "PD-Cluster-Clinic", "title": "Store leader score", "group": "Statistics - balance", "used_for": "Used for monitoring the leader score of each TiKV store, which represents the store's capacity to host region leaders based on its load and available resources. Essential for diagnosing leader distribution imbalances and understanding why PD schedules leaders to specific stores during load balancing operations.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_score\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 8, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances and tracking connection token usage. Essential for diagnosing connection exhaustion issues, identifying connection leaks, and determining if the cluster is approaching connection limits that could cause new connection failures.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 354, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive message batch bytes percentiles", "group": "Peer Messages", "used_for": "Used for monitoring the size distribution of message batches received by TiCDC's message server, tracking p50, p95, and p99 percentiles. Essential for diagnosing network communication efficiency and identifying abnormally large message batches that could indicate replication bottlenecks or memory pressure in peer-to-peer data synchronization.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 613, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output DDL Count & Executing Duration", "group": "Sink - General", "used_for": "Used for monitoring TiCDC's DDL statement execution rate and p999 latency at the sink layer. Critical for diagnosing DDL replication delays and identifying slow schema changes that may block data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4287, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's unified read pool threads across instances. Essential for diagnosing read performance bottlenecks and identifying instances with high CPU consumption during read-heavy workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified_read_po.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "Dedicated Dataplane Node Info", "title": "Notice", "group": "", "used_for": "**Used For:**\n\nUsed for displaying important notices, warnings, or configuration information about the dedicated dataplane node. Provides critical context or alerts that operators should be aware of when monitoring this specific node's status and behavior.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 188, "dashboard": "TiDB-Cluster-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of open file descriptors in TiDB server processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause service disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 356, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive Message Bytes Percentile", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message sizes received by TiCDC's message server at different percentiles (p50, p95, p99). Helps identify abnormally large messages in peer communication that could indicate data replication issues or network bandwidth concerns.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12892, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Operation", "group": "Raft Engine", "used_for": "Used for monitoring the operation rates of Raft Engine I/O activities, including write apply, entry reads, and message reads. Helps identify I/O throughput patterns and diagnose performance bottlenecks in TiKV's Raft log storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_apply_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_read_entry_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_read_message_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiKV instances. Essential for identifying storage engine CPU bottlenecks and diagnosing performance issues related to data compaction, flushing, and I/O operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31115, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the wait time (p95, p99) before read index proposals are processed in TiKV's storage engine. Critical for diagnosing read latency issues caused by Raft read index proposal queuing, especially when strong consistency reads experience delays waiting for leader confirmation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4556, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the disk space consumed by TiKV's encryption metadata files. Helps identify storage overhead from encryption key management and detect potential issues with metadata file growth that could impact disk capacity.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "PD-Cluster-Clinic", "title": "Store Region count", "group": "Statistics - balance", "used_for": "Used for monitoring the distribution of regions across different TiKV stores in the cluster. Essential for identifying region imbalance issues that can lead to hotspots and uneven load distribution, helping diagnose performance bottlenecks caused by poor data placement.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 56, "dashboard": "PD-Cluster-Clinic", "title": "Store leader size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of Raft leader regions on each TiKV store. Essential for diagnosing leader distribution imbalances that can cause uneven load across stores and identifying stores that may be overloaded with leader responsibilities.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 46, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Duration 95", "group": "Statistics", "used_for": "Used for monitoring the 95th percentile duration of automatic statistics collection operations in TiDB. Critical for identifying performance issues with auto-analyze jobs that may impact query optimization and overall cluster performance during statistics updates.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 620, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Full Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction flush operations in TiCDC's sink component. Critical for diagnosing downstream write performance issues and identifying bottlenecks in data replication to target systems.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 16, "dashboard": "Dedicated Dataplane Node Info", "title": "Node Info", "group": "", "used_for": "Used for displaying basic node information in Kubernetes clusters, including node metadata and system details. Helps operators verify node identity, OS version, and cluster membership when troubleshooting infrastructure issues or validating node configurations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "instance", "k8s_cluster" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node,k8s_cluster_info) group_right kube_node_info" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 4557, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypt/decrypt data nanos", "group": "Encryption", "used_for": "Used for monitoring the time spent on encrypting and decrypting data in TiKV's RocksDB storage engine, broken down by request type. Critical for diagnosing encryption-at-rest performance overhead and identifying whether encryption operations are causing I/O bottlenecks or query slowdowns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"encrypt_data_nanos\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"decrypt_data_nanos\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12893, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration", "group": "Raft Engine", "used_for": "Used for monitoring the write latency of TiKV's Raft Engine, which persists Raft logs to disk. Critical for diagnosing slow Raft log writes that can cause replication delays, leader election issues, and overall cluster performance degradation, with p95/p99/p999 percentiles helping identify tail latency problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.95, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31116, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's storage engine, showing p95 and p99 percentiles. Critical for diagnosing read performance issues related to Raft read index verification, which ensures linearizable reads in distributed transactions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 205, "dashboard": "TiDB-Cluster-Clinic", "title": "Disconnection Count", "group": "Server", "used_for": "Used for tracking the total number of client disconnections from TiDB instances, categorized by disconnection result type. Essential for diagnosing connection stability issues, identifying abnormal client disconnection patterns, and troubleshooting network or client-side problems that may impact application availability.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 350, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Stream Count Between Nodes", "group": "Peer Messages", "used_for": "**Used For:**\n\nUsed for monitoring the current number of active message streams between TiCDC nodes in the cluster. Helps diagnose inter-node communication issues, connection stability problems, and potential network bottlenecks affecting data replication coordination.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "ticdc_message_server_cur_stream_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 47, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Queries Per Minute", "group": "Statistics", "used_for": "Used for monitoring the rate of automatic statistics collection operations in TiDB per minute, broken down by analysis type. Helps identify when auto-analyze jobs are running and diagnose potential performance impacts from excessive statistics gathering or detect when auto-analyze isn't triggering as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 618, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Conflict Detect Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction conflict detection in TiCDC's transaction sink, along with average detection time and queue duration. Critical for diagnosing replication delays caused by transaction conflicts when TiCDC writes to downstream databases, helping identify whether slowdowns stem from conflict resolution overhead or queue congestion.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 358, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Total Stream Count", "group": "Peer Messages", "used_for": "Used for monitoring the total number of active peer message streams in TiCDC and comparing it against the expected stream count based on running TiCDC instances. Helps identify stream connection issues, network problems, or TiCDC node communication failures when actual streams deviate from the expected count (2N-1 for N nodes).", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_message_server_cur_stream_count)", "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}) - 1" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1708, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring the CPU utilization rate of TiFlash-Proxy processes across the cluster. Essential for identifying CPU bottlenecks, resource contention, and determining if TiFlash-Proxy nodes are overloaded or require scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 19, "dashboard": "Lightning-Cluster-Clinic", "title": "SST process duration", "group": "", "used_for": "Used for monitoring the performance of SST (Sorted String Table) file processing during TiDB Lightning data import operations. Helps diagnose import bottlenecks by tracking SST chunk throughput, split duration, and upload latency to TiKV nodes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 14, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Response Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the rate of response data bytes per second returned by TiFlash coprocessor requests, broken down by request type. Helps identify data transfer bottlenecks and diagnose issues with query result size or network throughput between TiFlash and TiDB.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 9, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second (QPS) of coprocessor requests handled by TiFlash, broken down by request type. Essential for tracking TiFlash's analytical query workload and identifying sudden changes in request patterns or throughput bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages received by each TiFlash-Proxy instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster coordination issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 8, "dashboard": "Lightning-Cluster-Clinic", "title": "Failures", "group": "", "used_for": "Used for monitoring the number of table import failures in TiDB Lightning across the cluster. Critical for quickly identifying data migration issues and tracking which tables failed to import successfully.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", result=\"failure\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 6, "dashboard": "Lightning-Cluster-Clinic", "title": "External resources", "group": "", "used_for": "Used for monitoring the number of active external resources (KV encoders, importer engines, and open engine RPC calls) currently in use by TiDB Lightning during data import operations. Helps identify resource leaks or accumulation issues when resources are opened but not properly closed, which can lead to memory exhaustion or import failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 1616, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft message flushes per TiFlash-Proxy server instance. Helps identify network communication patterns and potential bottlenecks in Raft consensus message processing between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 5, "dashboard": "Lightning-Cluster-Clinic", "title": "Idle workers", "group": "", "used_for": "Used for monitoring the number of idle worker threads in TiDB Lightning during data import operations. Helps identify resource utilization efficiency and potential bottlenecks—too many idle workers may indicate I/O constraints or upstream data supply issues, while consistently zero idle workers suggests the import is CPU/worker-bound.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_idle_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 32, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs in TiFlash-Proxy per server instance. Critical for diagnosing replication lag and write performance bottlenecks when TiFlash nodes struggle to keep up with data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 216, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of Wait Reactor", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's Wait Reactor threads, which handle asynchronous I/O operations and event processing in the pipeline execution model. Essential for diagnosing whether wait reactors are becoming CPU bottlenecks or if thread pool sizing needs adjustment during query execution.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 31, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of applying Raft logs in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing replication lag and write performance issues when TiFlash nodes experience delays in applying committed log entries from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process rate", "group": "", "used_for": "Used for monitoring the throughput rate of SQL data delivery during TiDB Lightning import operations. Helps identify data ingestion bottlenecks and track the overall progress of bulk data loading into the cluster.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "lightning.json" }, { "panel_id": 63, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Cop task memory usage", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring memory consumption of coprocessor requests in TiFlash at various percentiles (p80-p999). Essential for identifying memory-intensive queries, detecting potential OOM risks, and optimizing TiFlash resource allocation when coprocessor tasks consume excessive memory.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "Lightning-Cluster-Clinic", "title": "Checksum progress", "group": "", "used_for": "Used for tracking the completion progress of table checksum verification during TiDB Lightning data import. Helps monitor how many tables have completed checksum validation versus the total number of tables, essential for ensuring data integrity and estimating remaining import time.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"pending\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 104, "dashboard": "DM-Professional-Clinic", "title": "Task State", "group": "", "used_for": "Used for monitoring the current state of DM (Data Migration) worker tasks across different sources and jobs. Essential for tracking task health, identifying stuck or failed migration tasks, and ensuring data replication is running smoothly.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 15, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process duration", "group": "", "used_for": "Used for monitoring the latency of SQL processing stages during TiDB Lightning data import operations. Helps diagnose performance bottlenecks by tracking row encoding duration and block delivery time, essential for optimizing bulk data loading efficiency.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 670, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region task worker pre-handle/generate snapshot CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's region task and worker threads responsible for pre-handling region operations and generating snapshots. Critical for diagnosing TiFlash region replication bottlenecks and identifying instances with high CPU consumption during snapshot generation or region data synchronization.", "component": "TIFLASH", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 10, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk process duration", "group": "", "used_for": "Used for monitoring the average time taken to process each data chunk during TiDB Lightning import operations. Essential for identifying import performance bottlenecks and estimating overall data loading completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "1/rate(lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 230, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Max Execute Time Per Round", "group": "Pipeline Model", "used_for": "Used for monitoring the maximum execution time of TiFlash pipeline tasks per scheduling round at p95 and p99 percentiles. Critical for identifying pipeline task scheduling inefficiencies, detecting tasks that monopolize execution time, and diagnosing performance bottlenecks in TiFlash's query execution pipeline.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1, "dashboard": "Lightning-Cluster-Clinic", "title": "Import speed", "group": "", "used_for": "Used for monitoring the data import throughput during TiDB Lightning operations, tracking write speed to TiKV, upload speed, and overall imported bytes per second. Essential for diagnosing import performance bottlenecks and ensuring data migration is proceeding at expected rates.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"imported\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"restored\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"written\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 3, "dashboard": "Lightning-Cluster-Clinic", "title": "CPU%", "group": "", "used_for": "Used for monitoring CPU utilization percentage of TiDB Lightning and Importer processes during data import operations. Essential for identifying resource bottlenecks, ensuring adequate CPU allocation, and diagnosing performance degradation during bulk data loading tasks.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}[2m])*100", "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}[2m])*100" ], "dashboard_file": "lightning.json" }, { "panel_id": 13, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Handle Duration", "group": "Coprocessor", "used_for": "Used for monitoring the latency distribution (p80, p95, p99, p999) of coprocessor request handling in TiFlash. Essential for diagnosing slow analytical queries and identifying performance degradation in TiFlash's request processing pipeline, particularly when users experience delays in table scan or aggregation operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Duration", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash coprocessor request latency at multiple percentiles (p80, p95, p99, p999). Essential for identifying slow analytical query processing and diagnosing performance degradation in TiFlash's columnar storage engine when handling pushed-down coprocessor requests from TiDB.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1215, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages exchanged between TiFlash-Proxy and PD (Placement Driver). Essential for diagnosing communication issues between TiFlash-Proxy and PD, which can indicate network problems or PD overload affecting cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1396, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiFlash-Proxy and PD (Placement Driver). Helps diagnose issues with region peer metadata synchronization and detect abnormal validation patterns that may indicate cluster topology problems or communication failures.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 228, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Status Change OPS", "group": "Pipeline Model", "used_for": "Used for monitoring the rate of pipeline task status transitions in TiFlash's pipeline execution model. Helps diagnose task scheduling efficiency and identify potential bottlenecks when tasks frequently change states, indicating possible resource contention or execution issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_pipeline_task_change_to_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\r\n" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 5, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft process", "used_for": "Used for monitoring the rate of Raft ready messages handled by TiFlash-Proxy's Raft store. Essential for diagnosing Raft consensus processing performance and identifying bottlenecks in TiFlash's replication pipeline when data synchronization from TiKV appears slow or stalled.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1615, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages sent from each TiFlash-Proxy instance to other nodes in the cluster. Helps identify communication bottlenecks, network issues, or instances with abnormally high/low message sending rates that could indicate replication problems or cluster imbalance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 153, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Sender", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's snapshot sender threads, which handle sending Raft snapshots to other replicas. Critical for diagnosing performance issues during region replication, data migration, or when new TiFlash nodes join the cluster.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 40, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations in TiFlash-Proxy per server instance. Critical for diagnosing write performance bottlenecks and identifying slow disk I/O issues in TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 224, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Thread Pool Size", "group": "Pipeline Model", "used_for": "Used for monitoring the size of task thread pools in TiFlash's pipeline execution model. Helps diagnose resource allocation issues and thread pool capacity problems when investigating query execution performance or I/O bottlenecks in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_task_thread_pool_size\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 77, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Handling Request Number", "group": "Coprocessor", "used_for": "Used for monitoring the number of coprocessor requests currently being processed by TiFlash instances, categorized by request type. Helps identify request processing bottlenecks and diagnose issues when TiFlash is handling high concurrent workloads or experiencing request queuing.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 21, "dashboard": "Lightning-Cluster-Clinic", "title": "Import/Normal mode", "group": "", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration across instances during Lightning import operations. Helps identify whether TiKV is in import mode (higher limit) or normal mode (lower limit), which is critical for diagnosing write performance issues and understanding cluster configuration state during bulk data imports.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "lightning.json" }, { "panel_id": 18, "dashboard": "Lightning-Cluster-Clinic", "title": "Deliver duration", "group": "", "used_for": "Used for monitoring the latency of data delivery operations during TiDB Lightning import processes. Critical for diagnosing bottlenecks in the data ingestion pipeline, including range delivery to TiKV, SST file delivery, and row-based KV pair delivery from Lightning to the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_row_kv_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_kv_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 100, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the number of active RPC threads in TiFlash instances handling coprocessor requests. Essential for diagnosing thread pool saturation, identifying instances under heavy RPC load, and troubleshooting coprocessor request processing bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4, "dashboard": "Lightning-Cluster-Clinic", "title": "Import Progress", "group": "", "used_for": "Used for monitoring the progress of data import operations in TiDB Lightning by tracking the ratio of finished chunks to estimated total chunks. Essential for estimating import completion time and identifying stalled or slow import jobs that may require intervention.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"estimated\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 1069, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of PD (Placement Driver) requests sent from TiFlash-Proxy instances. Helps identify communication patterns with PD and diagnose issues when TiFlash nodes experience problems fetching cluster metadata or region information.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Executor QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second handled by different TiFlash coprocessor executors, broken down by executor type (e.g., table scan, aggregation, join). Essential for understanding TiFlash query workload distribution and identifying which executor operations are most frequently used or experiencing throughput changes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 151, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Small Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background worker threads that handle small maintenance tasks like compaction and cleanup. Helps identify when background operations are consuming excessive CPU resources or when thread pool utilization is abnormal, which could impact foreground query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Messages", "group": "Raft message", "used_for": "Used for monitoring the rate of different types of Raft messages sent by TiFlash-Proxy nodes in the cluster. Helps diagnose Raft communication patterns and identify potential issues with consensus protocol message flow, such as excessive heartbeats, vote requests, or append entries that could indicate network problems or cluster instability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 156, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Large Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background threads handling large tasks like compaction and data reorganization. Essential for diagnosing performance issues related to background operations that may compete with foreground query processing resources.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 161, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Client", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async client threads that handle remote procedure calls. Essential for diagnosing communication bottlenecks between TiFlash and other TiDB components, particularly when experiencing high network request latency or thread saturation issues.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Error QPS", "group": "Coprocessor", "used_for": "Used for monitoring the rate of coprocessor request errors per second in TiFlash, broken down by error reason. Critical for identifying and diagnosing issues with query execution failures, helping pinpoint whether errors stem from timeout, region unavailability, or other coprocessor-related problems.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 39, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of appending Raft logs to disk in TiFlash-Proxy's Raft store, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication performance issues and identifying disk I/O bottlenecks that could slow down data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 13, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk parser read block duration", "group": "", "used_for": "Used for monitoring the average duration of chunk parser read block operations and I/O apply worker latency in TiDB Lightning during data import. Critical for diagnosing data ingestion performance bottlenecks and identifying slow file reading or parsing issues that may impact overall import speed.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_chunk_parser_read_block_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_chunk_parser_read_block_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_apply_worker_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) /rate(lightning_apply_worker_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 123, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "0.99 Duration of raft store events", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile latency of Raft store events in TiFlash-Proxy, tracking how long different Raft operations take to complete. Critical for diagnosing TiFlash replication delays and identifying performance bottlenecks in Raft log processing, particularly when investigating slow data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "Lightning-Cluster-Clinic", "title": "Number of Lightning Goroutines", "group": "", "used_for": "Used for monitoring the number of active goroutines in TiDB Lightning instances during data import operations. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact import performance or cause Lightning to hang.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 218, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of CPU Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's CPU task thread pool, which handles compute-intensive operations in the pipeline execution model. Essential for identifying CPU bottlenecks, thread pool saturation, and optimizing TiFlash query performance under heavy analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 165, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Exchange Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the data exchange rate (bytes per second) between TiFlash nodes during distributed query execution, broken down by exchange type. Essential for diagnosing network bottlenecks and data shuffling performance issues in MPP queries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_exchange_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1070, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different types of requests from TiFlash-Proxy to PD (Placement Driver). Critical for diagnosing communication delays between TiFlash compute nodes and PD, which can impact metadata operations and region information updates.", "component": "PD", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3690, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of committing Raft logs to disk in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing write path performance issues and identifying when disk I/O becomes a bottleneck in TiFlash's Raft replication process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3688, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations in TiFlash-Proxy, broken down by server instance. Critical for diagnosing write path performance issues and identifying slow commit operations that could impact data replication and consistency in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 226, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Count", "group": "Pipeline Model", "used_for": "Used for monitoring the number of pipeline tasks in TiFlash's execution engine, showing both per-instance and cluster-wide task counts. Essential for diagnosing TiFlash query execution bottlenecks, identifying task accumulation issues, and understanding workload distribution across instances.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (instance, type)", "sum(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 267, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Scheduler", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's segment scheduler threads, which manage data segment operations like merging and splitting. Essential for diagnosing performance bottlenecks in TiFlash's background maintenance tasks and identifying when segment scheduling becomes a resource constraint.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 222, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Duration", "group": "Pipeline Model", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of pipeline tasks in TiFlash's pipeline execution model. Essential for identifying performance bottlenecks in TiFlash query processing and diagnosing slow task execution issues that impact overall query latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m]))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "Lightning-Cluster-Clinic", "title": "Memory usage", "group": "", "used_for": "Used for monitoring memory consumption of TiDB Lightning and its importer component during data import operations. Critical for detecting memory leaks, preventing OOM issues, and ensuring sufficient resources are allocated for large-scale data migration tasks.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 268, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Reader", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SegmentReader threads, which handle reading data from storage segments during query execution. Essential for identifying compute bottlenecks in TiFlash's columnar storage layer and diagnosing performance issues when scanning large datasets.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 118, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Process ready duration per server", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile duration of Raft ready state processing in TiFlash-Proxy per server. Critical for diagnosing Raft replication latency issues and identifying servers experiencing slow consensus processing that could impact TiFlash data synchronization performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 220, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of IO Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's IO task thread pool, which handles asynchronous I/O operations. Essential for identifying CPU bottlenecks in I/O processing and determining if the IO thread pool is saturated or underutilized during heavy read/write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 159, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Server", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async server threads that handle asynchronous network operations. Critical for diagnosing network communication bottlenecks and identifying when async polling threads are saturated or underutilized relative to available CPU cores.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 246, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "TiFlash Resource Group", "group": "TiFlash Resource Control", "used_for": "Used for monitoring TiFlash resource group consumption and throttling behavior, tracking remaining tokens, average consumption speed, and total resource usage per group. Essential for diagnosing resource control effectiveness, identifying resource-constrained groups, and ensuring fair resource allocation across different workloads in TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"remaining_tokens\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"avg_speed\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"total_consumption\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_fill_rate\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_capacity\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fetch_tokens_from_gac_count\", instance=~\"$instance\"}) by (instance,resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 149, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Manual Compaction", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's manual compaction thread pool. Helps identify when manual compaction operations are consuming excessive CPU resources, which may indicate heavy data reorganization workloads or potential performance bottlenecks in storage layer maintenance tasks.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "Lightning-Cluster-Clinic", "title": "Total bytes", "group": "", "used_for": "Used for monitoring data volume metrics during TiDB Lightning import operations, including bytes read from source data, bytes delivered to TiKV, and total cluster storage size. Essential for tracking import progress, estimating completion time, and verifying that imported data volume matches expected storage consumption.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_row_read_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"max_replicas\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 107, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Min TSO", "group": "Task Scheduler", "used_for": "Used for monitoring the minimum Timestamp Oracle (TSO) value being processed by TiFlash's task scheduler across different resource groups. Helps identify data freshness issues and diagnose replication lag between TiKV and TiFlash, as lower TSO values indicate older data being processed.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"min_tso\"}) by (instance, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 96, "dashboard": "DM-Professional-Clinic", "title": "Chunk process duration", "group": "dump and load ", "used_for": "Used for monitoring the average time taken to process each data chunk during DM's dump and load operations. Essential for identifying performance bottlenecks in data migration tasks and diagnosing slow chunk processing that could delay overall migration completion.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "1/rate(lightning_chunks{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",state=\"finished\"}[2m])" ], "dashboard_file": "dm.json" }, { "panel_id": 2741, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiFlash-Proxy instances, categorized by error type. Essential for detecting severe operational issues that could impact TiFlash's ability to replicate and serve data, requiring immediate investigation and intervention.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1710, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring disk I/O utilization percentage across TiFlash-Proxy nodes. Essential for identifying disk bottlenecks and I/O saturation issues that could impact TiFlash query performance and data synchronization.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1975, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft log speed", "group": "Raft propose", "used_for": "Used for monitoring the average rate of Raft log data being proposed in TiFlash-Proxy instances, measured in bytes per second. Helps identify write throughput patterns and potential bottlenecks in Raft log replication to TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 25, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Vote", "group": "Raft message", "used_for": "**Used For:**\n\nUsed for monitoring the rate of Raft vote messages sent by TiFlash-Proxy instances during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 95, "dashboard": "DM-Professional-Clinic", "title": "checksum progress", "group": "dump and load ", "used_for": "Used for monitoring the checksum verification progress during DM's data migration process. Shows the ratio of completed table checksums to pending checksums, helping users track validation progress and identify if checksum verification is stalling or completing as expected.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"completed\"} / ignoring(state) lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"pending\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1719, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring error rates in TiFlash-Proxy's scheduler component, specifically tracking snapshot and prepare write errors. Critical for diagnosing TiFlash-Proxy scheduling failures that could impact data replication and query execution between TiFlash and TiKV.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1309, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft message", "used_for": "Used for monitoring the rate of dropped Raft messages in TiFlash-Proxy, categorized by message type. Critical for diagnosing Raft communication issues, network problems, or overload conditions that prevent proper consensus protocol operation between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1721, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by instance and message type. Critical for identifying communication failures between TiFlash and TiKV that could impact data replication and query execution.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed in TiFlash-Proxy's raftstore. Critical for diagnosing replication delays and identifying bottlenecks in the Raft consensus pipeline when TiFlash experiences slow data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 89, "dashboard": "DM-Professional-Clinic", "title": "External resources", "group": "dump and load ", "used_for": "Used for monitoring the number of active external resources (KV encoders and importer engines) during DM's dump and load operations. Helps identify resource leaks or bottlenecks by tracking the difference between opened and closed resources in Lightning's data import process.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}", "lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 2536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied in TiFlash-Proxy's Raftstore, broken down by server instance. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 32, "dashboard": "DM-Professional-Clinic", "title": "dump process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's dump process over 30-minute intervals. Critical for identifying dump task failures and data migration interruptions that require immediate attention.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_dumpling_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 113, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Tasks Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting tasks in TiFlash's task scheduler across different resource groups. Helps identify task queue bottlenecks, resource contention, and scheduling inefficiencies when tasks accumulate in waiting state or when active task counts approach capacity limits.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_tasks_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_tasks_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 199, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Time of the Longest Live MPP Task", "group": "Coprocessor", "used_for": "Used for monitoring the execution time of the longest-running MPP (Massively Parallel Processing) task currently active in TiFlash. Helps identify stuck or abnormally slow MPP queries that may be consuming excessive resources or experiencing performance issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_mpp_task_monitor{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "DM-Professional-Clinic", "title": "load progress", "group": "dump and load ", "used_for": "Used for monitoring the data loading progress in DM's lightning component during the dump and load phase. Shows the ratio of finished chunks to estimated total chunks, helping users track migration progress and estimate completion time for the load operation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"} / ignoring(state) lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"estimated\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 33, "dashboard": "DM-Professional-Clinic", "title": "load process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's load process during data migration. Helps identify and diagnose failures in the data loading phase by tracking the rate of abnormal process terminations over 30-minute intervals.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_loader_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 119, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft read proposals (including local reads and read index operations) per TiFlash instance. Helps identify read request distribution across TiFlash nodes and diagnose potential read hotspots or imbalanced read workloads in the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 38, "dashboard": "DM-Professional-Clinic", "title": "Replication lag", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile replication lag in DM (Data Migration) binlog synchronization tasks. Critical for identifying data synchronization delays between upstream and downstream databases, helping diagnose replication bottlenecks and ensuring data consistency within acceptable time windows.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_lag_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,source_id,task))", "dm_syncer_replication_lag_gauge{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 76, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft admin", "used_for": "Used for monitoring the rate of Raft administrative operations (configuration changes and leader transfers) in TiFlash-Proxy. Essential for tracking cluster topology changes and diagnosing issues related to region leadership transfers or membership reconfigurations that may impact TiFlash's data replication and availability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 101, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the maximum number of RPC threads in TiFlash instances. Helps identify thread pool saturation and potential bottlenecks in handling coprocessor requests, which can indicate overload conditions or the need for thread pool tuning.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiFlash-Proxy, broken down by instance and error reason. Essential for diagnosing query execution failures and identifying problematic TiFlash nodes or specific error patterns affecting analytical workloads.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "DM-Professional-Clinic", "title": "SQL process rate", "group": "dump and load ", "used_for": "Used for monitoring the SQL data processing throughput rate during DM's dump and load operations. Helps identify data ingestion bottlenecks and track the speed at which SQL statements are being delivered and processed across different tasks and sources.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 2535, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft log entries are applied in TiFlash-Proxy's raftstore, tracking p95, p99, and average latencies. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply OPM", "group": "DDL", "used_for": "Used for monitoring the rate of schema changes (DDL operations) being applied to TiFlash instances, measured in operations per minute. Helps identify DDL activity patterns and diagnose issues when schema synchronization between TiDB and TiFlash is slow or failing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 37, "dashboard": "DM-Professional-Clinic", "title": "Remaining time to sync", "group": "binlog replication", "used_for": "Used for monitoring the estimated time remaining for DM (Data Migration) to catch up with the upstream binlog replication lag. Critical for tracking replication progress and identifying when sync delays may impact data freshness or recovery objectives.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_remaining_time{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiFlash-Proxy's Raftstore engine. Critical for detecting storage layer errors and replication issues that could impact TiFlash's data consistency and query performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 103, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads", "group": "Coprocessor", "used_for": "Used for monitoring the maximum thread count limits configured in TiFlash instances across different thread pools. Helps identify thread pool capacity constraints and potential resource bottlenecks when actual thread usage approaches these maximum thresholds.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1714, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the total number of Raft regions managed by each TiFlash-Proxy instance. Essential for tracking region distribution across TiFlash nodes and diagnosing data skew or unbalanced region allocation issues that could impact query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring the uptime duration of TiFlash-Proxy instances since their last restart. Helps identify recent restarts, instance stability issues, and verify successful deployments or rolling updates.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - tiflash_proxy_process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft Entry Cache", "group": "Cluster", "used_for": "Used for monitoring memory usage of the Raft entry cache in TiFlash-Proxy's raftstore. Helps diagnose memory pressure issues and identify potential memory leaks in the Raft log replication layer when TiFlash nodes experience high memory consumption.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 19, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Internal DDL OPM", "group": "DDL", "used_for": "Used for monitoring the rate of internal DDL operations per minute in TiFlash, broken down by operation type and instance. Essential for tracking schema synchronization activity between TiDB and TiFlash, helping diagnose DDL replication delays or excessive schema change overhead in the columnar storage engine.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 109, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Estimated Thread Usage and Limit", "group": "Task Scheduler", "used_for": "Used for monitoring TiFlash's task scheduler thread consumption against configured soft and hard limits per resource group. Critical for diagnosing thread exhaustion issues, identifying resource group contention, and preventing task scheduler overload that could impact query execution performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_soft_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_hard_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"global_estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"group_entry_count\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 115, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Waiting Duration", "group": "Task Scheduler", "used_for": "Used for monitoring the waiting time of tasks in TiFlash's task scheduler at p80, p90, and p100 percentiles. Essential for diagnosing query queuing issues and resource contention when TiFlash queries experience delays before execution begins.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(1.00, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 157, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "MPP Query count", "group": "Coprocessor", "used_for": "Used for monitoring the count of MPP (Massively Parallel Processing) queries being executed in TiFlash instances. Helps identify query workload distribution across TiFlash nodes and diagnose issues when MPP query execution is imbalanced or abnormally high.", "component": "TIFLASH", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 111, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Queries Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting queries in TiFlash's task scheduler across different resource groups. Helps identify query queuing issues, resource contention, and whether TiFlash is overloaded or experiencing scheduling bottlenecks that could impact query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_queries_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_queries_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft propose", "used_for": "Used for monitoring the rate of different types of Raft proposals in TiFlash-Proxy, including read operations (local_read, read_index) and write operations (normal, batch). Essential for diagnosing I/O bottlenecks and understanding the distribution of read/write workload patterns in TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Check split", "group": "Raft admin", "used_for": "Used for monitoring the rate of region split checks in TiFlash-Proxy's Raft store, categorized by check type. Helps diagnose region split behavior and identify potential issues with region management or split decision-making in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 110, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS (Sum by task)", "group": "binlog replication", "used_for": "Used for monitoring the binlog replication throughput in DM (Data Migration) tasks, showing both actual finished jobs per second and ideal QPS aggregated by task. Essential for identifying replication performance bottlenecks, comparing actual versus expected throughput, and diagnosing slow data synchronization issues across different migration tasks.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "task" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (task)", "sum(dm_syncer_ideal_qps{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 1584, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiFlash-Proxy server overload conditions, including scheduler saturation, channel congestion, and coprocessor request rejections due to capacity limits. Critical for diagnosing when TiFlash-Proxy instances are overwhelmed and unable to handle incoming requests, helping identify resource bottlenecks or excessive workload that may require scaling or optimization.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4375, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile latency of performance context operations during Raft propose in TiFlash-Proxy, including both store and apply phases. Critical for diagnosing slow Raft log replication and identifying performance bottlenecks in TiFlash's consensus layer that may impact write throughput.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile of Raft proposals processed per ready batch in TiFlash-Proxy. Helps diagnose Raft replication efficiency and identify instances where proposal batching may be suboptimal, potentially indicating I/O bottlenecks or apply worker congestion.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1709, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring the resident memory usage of TiFlash-Proxy processes across cluster instances. Essential for tracking memory consumption patterns, detecting memory leaks, and ensuring TiFlash-Proxy operates within allocated resource limits.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft admin", "used_for": "Used for monitoring the rate of successful Raft admin commands in TiFlash-Proxy, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes that affect TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "DM-Professional-Clinic", "title": "dump progress", "group": "dump and load ", "used_for": "Used for monitoring the data dump progress in DM (Data Migration) tasks by showing the ratio of finished rows to estimated total rows. Helps track dump operation completion status and estimate remaining time for the data export phase.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dumpling_dump_finished_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"} / dumpling_dump_estimate_total_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 18, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiFlash-Proxy nodes, categorized by failure type and instance. Critical for identifying communication issues between TiFlash-Proxy and PD/TiKV that could indicate network problems, node unavailability, or cluster instability.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 166, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data size in send and receive queue", "group": "Coprocessor", "used_for": "Used for monitoring the amount of data queued in TiFlash's exchange operators during coprocessor request processing. Helps identify data transfer bottlenecks and memory pressure when TiFlash nodes exchange intermediate results during distributed query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_exchange_queueing_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 39, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "binlog replication", "used_for": "Used for monitoring abnormal process terminations in DM's binlog replication component. Helps quickly identify and diagnose replication failures by tracking error-triggered exits over 30-minute intervals, essential for maintaining data migration stability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_syncer_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 18, "dashboard": "DM-Professional-Clinic", "title": "Idle workers", "group": "dump and load ", "used_for": "Used for monitoring the number of idle worker threads in DM's lightning component during data dump and load operations. Helps identify resource utilization issues, such as insufficient parallelism (too many idle workers) or potential bottlenecks preventing workers from processing data efficiently.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_idle_workers{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1722, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region drops in TiFlash-Proxy instances. Critical for detecting leadership instability, network partitions, or node failures that cause regions to lose their leader status, which can impact data availability and query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 102, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS", "group": "binlog replication", "used_for": "Used for monitoring the throughput of binlog replication operations in DM (Data Migration), measured as queries per second. Helps identify replication performance bottlenecks and track the rate of different operation types (INSERT/UPDATE/DELETE) being replicated from source to target databases.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval]))by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 1723, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions in TiFlash-Proxy that are missing their leaders. Critical for diagnosing cluster stability issues, as missing leaders indicate network partitions, node failures, or Raft consensus problems that prevent data replication and query processing.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 41, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft proposals are processed in TiFlash-Proxy's raftstore, showing p95, p99, and average latencies. Critical for diagnosing Raft replication bottlenecks and identifying when proposals are queuing excessively before being handled, which can indicate raftstore thread saturation or processing delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 102, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads", "group": "Coprocessor", "used_for": "Used for monitoring the current thread count across different thread pools in TiFlash instances, excluding RPC and maximum limit threads. Essential for identifying thread resource exhaustion, detecting thread leaks, and diagnosing I/O bottlenecks in TiFlash's coprocessor operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "DM-Professional-Clinic", "title": "Total bytes for load unit", "group": "dump and load ", "used_for": "Used for monitoring the total data volume processed during DM's load phase, tracking both bytes read from source files and bytes delivered to TiDB. Essential for understanding data migration progress, estimating completion time, and diagnosing throughput bottlenecks in the load unit.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_row_read_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", "lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 41, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of write operations in TiFlash storage engine, tracking both storage commands and DeltaTree write block events per second. Essential for identifying write workload patterns, detecting write traffic spikes, and diagnosing TiFlash write performance bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL schema changes being applied to TiFlash replicas at various percentiles (p80-p999). Critical for diagnosing slow DDL propagation issues and identifying when schema synchronization between TiDB and TiFlash becomes a bottleneck, which can impact query performance on analytical workloads.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "type" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", keyspace!=\"\"}[2m])) by (le, type, keyspace))", "sum(tiflash_sync_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"$type\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 120, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiFlash instance. Helps identify write workload distribution across TiFlash nodes and diagnose imbalanced write traffic or instances experiencing abnormally high proposal rates that could indicate performance bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 71, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiFlash's Raft proxy layer. Critical for identifying extreme tail latency in split operations that could indicate storage bottlenecks or resource contention affecting TiFlash's data distribution and rebalancing performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 33, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "Used for monitoring the total storage size of different column families (CFs) in TiFlash-Proxy's RocksDB engine. Helps track storage usage patterns and identify potential disk space issues or data distribution imbalances across column families.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1715, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the number of Raft leader regions on each TiFlash node and detecting significant leader transfers. Helps identify load imbalance across TiFlash instances and diagnose unexpected leader migration events (drops of more than 10 leaders within 2 minutes), which may indicate node instability or cluster rebalancing issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2, "dashboard": "DM-Professional-Clinic", "title": "DM SQL jobs ", "group": "binlog replication", "used_for": "Used for monitoring the rate of SQL job processing in DM's binlog replication pipeline, showing both completed jobs (finished) and incoming jobs (added) by task, source, schema, and operation type. Essential for identifying replication bottlenecks, job queue buildup, and understanding the throughput of different SQL operation types during data migration.", "component": "", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", "sum(-rate(dm_syncer_added_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 4, "dashboard": "DM-Professional-Clinic", "title": "binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being processed per second during DM (Data Migration) replication. Essential for tracking replication throughput and identifying bottlenecks or slowdowns in the binlog transformation pipeline across different tasks and stages.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "interval", "job_id", "source", "stage", "task" ], "promql": [ "sum by (instance,task,stage) (rate(dm_syncer_binlog_transform_cost_count{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\", source_id=~\"$source\",stage=~\"$stage\"}[$interval]))" ], "dashboard_file": "dm.json" }, { "panel_id": 64, "dashboard": "DM-Professional-Clinic", "title": "DML conflict detect duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of DML conflict detection during binlog replication in DM (Data Migration) tasks at p90, p95, and p99 percentiles. Critical for diagnosing replication slowdowns caused by conflict detection overhead, especially when multiple concurrent writes target the same data and require conflict resolution.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 2, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the rate of different storage command types executed in TiFlash-Proxy. Helps identify command distribution patterns and diagnose issues when specific storage operations (like get, scan, prewrite) show abnormal rates or spikes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 40, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the rate of read tasks processed per second by each TiFlash instance. Helps identify read workload distribution across TiFlash nodes and diagnose performance bottlenecks when read throughput is unbalanced or unexpectedly low.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiFlash's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate workload imbalances or priority queue bottlenecks affecting read performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1705, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the total storage size of each TiFlash-Proxy instance's RocksDB engine. Essential for tracking disk space usage, capacity planning, and identifying instances approaching storage limits that may require scaling or data cleanup.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 63, "dashboard": "DM-Professional-Clinic", "title": "add job duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of adding binlog replication jobs to DM syncer queues at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when job queuing delays impact replication performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "queueNo", "source", "task", "type" ], "promql": [ "histogram_quantile(0.90, sum (rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.95, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.99, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))" ], "dashboard_file": "dm.json" }, { "panel_id": 58, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region in TiFlash-Proxy instances. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 15, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing slow snapshot generation issues that can impact TiFlash's ability to serve consistent reads and replicate data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 35, "dashboard": "DM-Professional-Clinic", "title": "skipped binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being skipped during DM (Data Migration) replication, broken down by rows and query types. Essential for identifying data filtering effectiveness and diagnosing potential data synchronization gaps when binlog filter rules are applied.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"rows\"}[$interval])", "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"query\"}[$interval])" ], "dashboard_file": "dm.json" }, { "panel_id": 3834, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the volume of data being written by the scheduler in TiFlash-Proxy instances. Helps identify write bottlenecks, uneven write distribution across instances, and potential memory pressure issues in the scheduler's write pipeline.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 8, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the error rate of asynchronous storage requests in TiFlash-Proxy's storage engine. Critical for identifying storage layer failures, request timeouts, or engine errors that could impact TiFlash query performance and data synchronization.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 60, "dashboard": "DM-Professional-Clinic", "title": "transaction/ddl query execution latency", "group": "binlog replication", "used_for": "Used for monitoring the P90, P95, and P99 latency of transaction and DDL query execution during binlog replication in DM (Data Migration). Critical for identifying replication lag issues and diagnosing slow transaction processing that could impact data synchronization performance between upstream and downstream databases.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.90, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 22, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "File Open OPS", "group": "Storage", "used_for": "Used for monitoring the rate of file open operations and failures in TiFlash storage layer. Helps diagnose file descriptor exhaustion, storage I/O issues, or excessive file operations that could impact TiFlash performance and stability.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3638, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiFlash-Proxy's Raft store. Helps identify region size imbalances and diagnose issues related to region splitting, merging, or uneven data distribution that could impact I/O performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 167, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the total number of scheduler operations across different stages in TiFlash-Proxy, including both normal operations and \"too busy\" rejections. Essential for diagnosing scheduler bottlenecks and understanding when TiFlash-Proxy is overloaded or rejecting requests due to resource constraints.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 255, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (rows)", "group": "Storage", "used_for": "Used for monitoring the row-level write throughput of TiFlash storage subtasks, broken down by operation type. Essential for analyzing TiFlash write performance and identifying bottlenecks in data ingestion from TiKV to the columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 42, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum latency of internal storage subtasks in TiFlash, such as delta merge operations and background compaction tasks. Critical for diagnosing TiFlash storage engine performance issues and identifying bottlenecks in background maintenance operations that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 38, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Amplification", "group": "Storage", "used_for": "Used for monitoring the write amplification ratio in TiFlash storage, which measures how much physical data is written to disk compared to logical data written by applications. Critical for identifying inefficient storage patterns, excessive compaction overhead, and I/O performance degradation that can impact TiFlash query performance and disk lifespan.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[10m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[30m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4198, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiFlash-Proxy's unified read pool at any given time. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient thread resources or slow query processing.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 50, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Opened File Count", "group": "Storage", "used_for": "Used for monitoring the number of open file descriptors in TiFlash instances, including total open files, files opened for writing, and files opened for reading. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiFlash crashes or performance degradation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4196, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of tasks being scheduled at level 0 (highest priority) in TiFlash's unified read pool. Helps diagnose read request prioritization issues and identify when the system is consistently processing high-priority requests, which may indicate resource contention or workload imbalance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 57, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region in TiFlash-Proxy. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing TiFlash write performance bottlenecks and identifying when data ingestion from TiKV experiences delays or degradation.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1481, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the approximate size distribution of Raft regions stored in TiFlash-Proxy, showing p95, p99, and average values. Essential for identifying oversized regions that may cause I/O bottlenecks, uneven data distribution, or performance degradation in TiFlash's columnar storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiFlash-Proxy. Helps identify priority distribution of I/O operations and diagnose whether high-priority commands are being properly processed or if low-priority commands are overwhelming the scheduler.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3636, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the rate of bytes written to regions in TiFlash-Proxy's storage layer. Helps diagnose write throughput issues, identify heavy write workloads, and detect abnormal I/O patterns that may impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3637, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the rate of keys written to regions in TiFlash-Proxy's storage engine. Helps diagnose write throughput issues and identify regions experiencing high write activity or potential hotspots.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 46, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Write OPS", "group": "Storage", "used_for": "Used for monitoring disk write operations per second in TiFlash storage layer, including PSM (PageStorage Manager) write calls, page writes, and file descriptor writes. Essential for diagnosing TiFlash write throughput bottlenecks and identifying abnormal disk I/O patterns that may impact columnar storage performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 131, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum duration of large internal storage tasks in TiFlash, such as delta merge, delta merge background, and place index operations. Critical for diagnosing TiFlash storage layer performance issues and identifying when background maintenance tasks are taking unusually long to complete.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 117, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Hard Limit Exceeded Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of times TiFlash task scheduler has exceeded hard resource limits per instance and resource group. Critical for identifying resource contention issues and determining when TiFlash workloads are being throttled due to insufficient capacity allocation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hard_limit_exceeded_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 130, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of TiFlash's internal storage maintenance operations, including delta merge, segment merge, and segment split tasks. Essential for diagnosing TiFlash storage performance issues and understanding background task activity that affects query performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 253, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (bytes)", "group": "Storage", "used_for": "Used for monitoring the write throughput of TiFlash storage subtasks, broken down by operation type. Helps identify write performance bottlenecks and understand which storage operations are consuming the most bandwidth in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel full events in TiFlash-Proxy, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, message processing bottlenecks, and potential data replication delays between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 106, "dashboard": "DM-Professional-Clinic", "title": "binlog event row number", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile of row counts per binlog event during DM (Data Migration) replication. Helps identify binlog events with unusually large row batches that may cause replication lag or memory pressure in the sync unit.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_row_bucket{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source))" ], "dashboard_file": "dm.json" }, { "panel_id": 1, "dashboard": "DM-Professional-Clinic", "title": "statement execution latency - 90", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile execution latency of binlog replication statements in DM (Data Migration), broken down by transaction phases (begin, statement execution, and commit). Critical for identifying performance bottlenecks in the replication pipeline and diagnosing slow transaction processing that could cause replication lag.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"begin\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"stmt\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"commit\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"rollback\", instance=~\"$instance\"}[2m])) by (le,task,instance))" ], "dashboard_file": "dm.json" }, { "panel_id": 45, "dashboard": "DM-Professional-Clinic", "title": "unsynced tables", "group": "binlog replication", "used_for": "Used for monitoring the number of tables that have not yet been synchronized in DM (Data Migration) binlog replication tasks. Critical for identifying replication lag issues and tracking which tables are falling behind in the data migration process.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_unsynced_table_number{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 193, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands in TiFlash-Proxy instances. Helps identify scheduling bottlenecks and command queue buildup that could indicate resource contention or processing delays in TiFlash's Raft proxy layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of incoming requests to TiFlash-Proxy, showing both average and p99 percentile values by request type. Helps identify request batching efficiency and detect anomalies where requests aren't being properly batched, which could impact TiFlash query performance and resource utilization.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 107, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by message type. Essential for identifying communication issues between TiFlash and other cluster components, helping diagnose network problems or service degradation.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 52, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FSync Status", "group": "Storage", "used_for": "Used for monitoring file synchronization (fsync) operations in TiFlash storage, tracking both the rate of fsync calls and their latency distribution. Critical for diagnosing disk I/O performance issues and ensuring data durability, as excessive fsync latency can indicate storage bottlenecks affecting TiFlash write performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 39, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of small internal storage tasks in TiFlash, excluding major operations like delta merge, segment merge, and segment split. Helps identify the activity level of background maintenance tasks and diagnose potential bottlenecks in TiFlash's internal storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 58, "dashboard": "DM-Professional-Clinic", "title": "binlog event cost duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of binlog event transformation at different stages (p90, p95, p99 percentiles) in DM's data replication pipeline. Essential for diagnosing binlog processing bottlenecks and identifying which transformation stage is causing replication delays when data migration performance degrades.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))" ], "dashboard_file": "dm.json" }, { "panel_id": 36, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of reading binlog events from upstream MySQL/MariaDB sources in DM (Data Migration) tasks, showing p90, p95, and p99 percentiles. Critical for diagnosing binlog replication delays and identifying slow binlog read operations that could bottleneck data synchronization performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 95, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages processed by TiFlash-Proxy, broken down by message type. Essential for identifying communication patterns between TiFlash and TiKV, and diagnosing issues with inter-component RPC traffic or message processing bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 168, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiFlash-Proxy's transaction processing pipeline. Helps identify bottlenecks in specific scheduler stages (like snapshot, async_write, etc.) and detect when the scheduler becomes too busy to handle requests, which can indicate resource contention or overload issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch ratio", "group": "Server", "used_for": "Used for monitoring the batch processing efficiency of requests in TiFlash-Proxy, showing both average and P99 batch ratios by request type. Essential for identifying request batching inefficiencies that could impact I/O performance and diagnosing whether requests are being properly grouped for optimal throughput.", "component": "TIFLASH", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 62, "dashboard": "DM-Professional-Clinic", "title": "DML queue remain length", "group": "binlog replication", "used_for": "Used for monitoring the number of DML (Data Manipulation Language) events queued and waiting to be replicated in DM's binlog synchronization pipeline. Critical for identifying replication bottlenecks and backlog issues when the queue length grows, indicating that DM cannot process binlog events fast enough to keep up with upstream changes.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "task" ], "promql": [ "sum(dm_syncer_queue_size{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler latch waiting in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing transaction conflicts and contention issues when multiple transactions compete for the same keys, helping identify performance bottlenecks caused by lock waiting in TiFlash's write path.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler commands in TiFlash-Proxy at p95 and p99 percentiles. Essential for diagnosing performance bottlenecks in TiFlash's transaction scheduling layer and identifying slow command execution that may impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 59, "dashboard": "DM-Professional-Clinic", "title": "dispatch binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of dispatching binlog events in DM (Data Migration) replication tasks at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when event dispatch delays are impacting replication lag between source and target databases.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 98, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message handling in TiFlash-Proxy. Critical for identifying communication bottlenecks between TiFlash and TiKV, helping diagnose slow data replication or query execution issues caused by inter-component network delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 60, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write flow", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash's write throughput across different write paths, including file descriptor writes, PageStorage (PSM) writes, and background writes. Essential for diagnosing TiFlash storage I/O bottlenecks, understanding write workload distribution, and identifying abnormal write patterns that may impact data ingestion performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 75, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the number of TiFlash-Proxy raft regions actively receiving write operations per instance. Helps identify write hotspots, uneven data distribution across TiFlash nodes, and diagnose performance issues related to write-heavy workloads in the TiFlash storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 47, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Read OPS", "group": "Storage", "used_for": "Used for monitoring the rate of disk read operations in TiFlash storage layer, including PSM (PageStorage Manager) read I/O calls, page reads, and file descriptor reads. Essential for diagnosing TiFlash read performance bottlenecks and identifying excessive disk I/O activity that may impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 107, "dashboard": "DM-Professional-Clinic", "title": "Flushed Checkpoint Time Interval", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile time interval between checkpoint flushes during DM binlog replication. Helps diagnose checkpoint persistence delays and identify tasks with irregular or slow checkpoint update patterns that could impact data consistency and recovery capabilities.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_flush_checkpoints_time_interval_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task,worker))" ], "dashboard_file": "dm.json" }, { "panel_id": 43, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM syncer during data migration. Critical for identifying replication lag and potential data synchronization delays, helping diagnose whether the syncer is keeping up with master's binlog generation rate.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"master\"} - ON(task,source_id) dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 195, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose I/O efficiency issues and identify commands that scan excessive keys, which may indicate inefficient queries or potential performance bottlenecks in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2534, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages sent through gRPC in TiFlash-Proxy. Helps diagnose message batching efficiency and identify potential network communication issues when batch sizes are abnormally large or small, which could impact replication performance between TiFlash and TiKV nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 67, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Current Data Management Tasks", "group": "Storage", "used_for": "Used for monitoring the number of active data management tasks in TiFlash's DeltaTree storage engine, including delta-merge operations, segment splits, and segment merges. Essential for understanding TiFlash's background compaction workload and diagnosing storage performance issues or resource contention during data reorganization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 61, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the size distribution of binlog events being replicated by DM (Data Migration) workers at p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that may cause replication delays, memory pressure, or network bottlenecks during data synchronization from MySQL to TiDB.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,worker,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 59, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read flow", "group": "Storage", "used_for": "Used for monitoring the read I/O throughput in TiFlash storage layer, including file descriptor reads, PageStorage (PSM) reads, and background reads. Essential for diagnosing read performance bottlenecks, identifying excessive disk I/O, and understanding storage layer read patterns during query execution.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 373, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose write amplification issues and identify commands that are writing unexpectedly large numbers of keys, which could impact TiFlash storage performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2532, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Average gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC messages between TiFlash and TiKV, broken down by message type. Essential for diagnosing communication bottlenecks and identifying which specific gRPC operations are experiencing delays in the TiFlash-TiKV data synchronization path.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 44, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between relay and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the relay log and syncer components in DM (Data Migration). Helps identify replication lag by showing how many binlog files the syncer is behind the relay, which is critical for diagnosing data synchronization delays and bottlenecks in the migration pipeline.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id", "task" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"} - ON(instance) group_right dm_syncer_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 560, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiFlash-Proxy's scheduler, broken down by operation tags. Helps diagnose scan performance issues and understand the characteristics of different scan types (e.g., processed keys, skipped keys) for specific scheduler commands.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2533, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile and average batch sizes of gRPC requests and responses in TiFlash-Proxy. Essential for diagnosing gRPC communication efficiency and identifying potential batching issues that could impact TiFlash query performance and network utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 46, "dashboard": "DM-Professional-Clinic", "title": "shard lock resolving", "group": "binlog replication", "used_for": "Used for monitoring the number of shard DDL locks currently being resolved in DM (Data Migration) during binlog replication. Critical for diagnosing coordination issues when merging sharded tables, as prolonged lock resolving indicates DDL synchronization delays or conflicts across multiple upstream MySQL instances.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_shard_lock_resolving{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 64, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's scheduler worker threads, which handle task scheduling and coordination. Essential for identifying scheduler bottlenecks, thread contention issues, or abnormal CPU consumption that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "DM-Professional-Clinic", "title": " transform binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of transforming binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing slow data transformation issues and identifying performance bottlenecks in the binlog processing pipeline that could cause replication lag.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 79, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's async apply threads, which handle Raft log application to the storage engine. Essential for diagnosing write performance bottlenecks and identifying whether apply threads are saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 84, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Throughput", "group": "Storage", "used_for": "Used for monitoring the I/O throughput controlled by TiFlash's I/O limiter, broken down by operation type (read/write). Essential for identifying I/O bottlenecks and verifying whether storage operations are being throttled due to configured I/O limits.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 829, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations on TiFlash-Proxy's write column family during scheduler command execution. Essential for diagnosing write-path performance issues and understanding scan patterns that may impact TiFlash write throughput.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 61, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash's Raft store threads, which handle Raft consensus protocol operations for data replication. Essential for diagnosing performance bottlenecks in TiFlash's data synchronization layer and identifying instances with excessive Raft processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 237, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexError", "group": "Storage", "used_for": "Used for monitoring the rate of delta index errors in TiFlash's DeltaTree storage engine. Critical for identifying data integrity issues or corruption in TiFlash's columnar storage that could affect query accuracy and system stability.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 30, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "relay log", "used_for": "Used for monitoring the count of DM relay log processes that have exited with errors. Critical for identifying relay log failures and data replication interruptions that require immediate attention.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_exit_with_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 830, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the distribution of key-value scan operations in TiFlash-Proxy's scheduler for the default column family. Helps diagnose scan performance issues by breaking down scan operations by tag, useful when investigating slow queries or identifying inefficient scan patterns in TiFlash storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 65, "dashboard": "DM-Professional-Clinic", "title": "skip event duration", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the latency of skipping binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing performance issues when DM filters or skips certain binlog events based on configured rules, helping identify if event filtering is causing replication delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 35, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent by TiFlash-Proxy nodes. Essential for tracking data replication activity and diagnosing issues with region snapshot transfers, which typically occur during node recovery, scaling operations, or when replicas fall significantly behind.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 266, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Pending Rate and Duration", "group": "Storage", "used_for": "Used for monitoring the rate of I/O operations waiting in TiFlash's I/O limiter queue, broken down by operation type. Essential for identifying I/O throttling issues and understanding when storage operations are being rate-limited, which can impact query performance and data synchronization latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter_pending_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4287, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's unified read pool threads across instances. Essential for identifying read workload bottlenecks and determining if the unified read pool requires resource adjustment when query performance degrades.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 269, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Scheduling", "group": "Storage", "used_for": "Used for monitoring TiFlash read thread scheduling behavior and resource utilization patterns. Helps diagnose read performance bottlenecks by tracking thread exhaustion, active segment limits, cache hits, and scheduling efficiency in the storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 36, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiFlash-Proxy, including sending, applying, and generating snapshots. Critical for diagnosing slow replica synchronization and identifying performance bottlenecks in TiFlash's Raft snapshot handling process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Snapshots", "group": "Storage", "used_for": "Used for monitoring the number of active read snapshots and segment read tasks in TiFlash storage engine. Essential for diagnosing MVCC snapshot accumulation issues, memory pressure from long-running queries, and identifying potential read performance bottlenecks caused by excessive concurrent read operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 132, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data Sharing", "group": "Storage", "used_for": "Used for monitoring TiFlash's data sharing cache performance, including cache hit rates, miss rates, and cache operation statistics. Essential for diagnosing read performance issues and evaluating the effectiveness of TiFlash's shared data cache mechanism across storage nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"add_cache_stale|add_cache_succ|add_cache_total_bytes_limit|add_cache_reach_count_limit\"}[2m])) by (type)", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))/sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 86, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Current Pending Count", "group": "Storage", "used_for": "Used for monitoring the number of write requests currently queued in TiFlash's I/O limiter, broken down by rate limiter, background writes, and foreground writes. Essential for diagnosing write throttling issues and identifying when storage I/O limits are causing request backlog in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 105, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC polling threads in TiFlash-Proxy instances. Essential for identifying excessive CPU consumption in gRPC communication layers, which may indicate network bottlenecks or inefficient request handling between TiFlash and other TiDB components.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2531, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's garbage collection worker threads. Essential for diagnosing GC performance issues and identifying instances where excessive GC activity may be impacting overall TiFlash performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 31, "dashboard": "DM-Professional-Clinic", "title": "relay log data corruption", "group": "relay log", "used_for": "Used for detecting data corruption events in DM (Data Migration) relay logs. Critical for identifying data integrity issues during binlog replication that could lead to inconsistent data migration or replication failures.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_data_corruption{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 59, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiFlash-Proxy. Helps identify worker thread activity patterns and diagnose task processing bottlenecks or imbalances across worker pools.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 20, "dashboard": "DM-Professional-Clinic", "title": "storage capacity", "group": "relay log", "used_for": "Used for monitoring the total storage capacity allocated for DM (Data Migration) relay logs. Helps administrators track available disk space for relay log storage and plan capacity upgrades before running out of space during data replication tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"capacity\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 38, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots by type (generating, applying, receiving, sending) in TiFlash-Proxy. Helps diagnose snapshot-related issues such as slow replica synchronization, excessive snapshot traffic, or bottlenecks in data replication between TiKV and TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 43, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pair counts in TiFlash-Proxy snapshots. Helps identify extreme cases where snapshots contain unusually large numbers of KV pairs, which can indicate data skew issues or potential performance bottlenecks during snapshot processing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and relay", "group": "relay log", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM's relay log. Helps identify replication lag at the file level and diagnose relay log synchronization delays or interruptions in data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"} - ON(instance) dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 168, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema of Column File", "group": "Storage", "used_for": "Used for monitoring TiFlash's column file schema cache performance, tracking cache size, hit rate, and eviction patterns. Essential for diagnosing schema cache efficiency issues and identifying when schema metadata is being evicted while still in use, which can impact query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)", "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[2m])) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1908, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash-Proxy's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether specific priority queues are saturated or experiencing CPU contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 169, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Effectiveness of Mark Cache", "group": "Storage", "used_for": "Used for monitoring the hit and miss rates of TiFlash's mark cache, which stores metadata about data block boundaries in columnar storage. Essential for evaluating cache effectiveness and diagnosing performance issues related to metadata lookups during query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_system_profile_event_MarkCacheMisses{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tiflash_system_profile_event_MarkCacheHits{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 78, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash coprocessor threads across different priority levels (normal, high, and low). Essential for diagnosing TiFlash query processing performance issues and identifying CPU bottlenecks in analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 67, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy background worker threads across instances. Essential for identifying background task overhead and diagnosing performance issues caused by maintenance operations like compaction, cleanup, or internal housekeeping tasks.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 62, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Stall Duration", "group": "Storage Write Stall", "used_for": "Used for monitoring the 99th percentile duration of write stalls in TiFlash's storage layer. Critical for diagnosing write performance degradation caused by storage pressure, helping identify when TiFlash instances experience backpressure that delays data ingestion from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, instance))", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 29, "dashboard": "DM-Professional-Clinic", "title": "fail to write relay log", "group": "relay log", "used_for": "Used for monitoring relay log write failures in DM (Data Migration) workers. Critical for diagnosing data replication interruptions caused by I/O errors when persisting binlog events to local relay logs.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_write_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 21, "dashboard": "DM-Professional-Clinic", "title": "storage remain", "group": "relay log", "used_for": "**Used For:**\n\nUsed for monitoring the available disk space remaining for DM relay logs. Critical for preventing relay log write failures due to insufficient storage and ensuring continuous data replication from upstream MySQL to downstream TiDB.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"available\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 69, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiFlash-Proxy instances. Essential for identifying CPU bottlenecks in the storage engine layer and diagnosing performance issues related to data compaction, writes, and reads in TiFlash's underlying RocksDB.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 28, "dashboard": "DM-Professional-Clinic", "title": "fail to read binlog from master", "group": "relay log", "used_for": "Used for monitoring the count of errors encountered when DM (Data Migration) relay log fails to read binlog from the upstream MySQL master. Critical for diagnosing replication interruptions and connectivity issues between DM relay and the source database.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_read_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1876, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different thread pools in TiFlash-Proxy's future pool. Helps identify thread pool utilization patterns and diagnose task processing bottlenecks or imbalances across different pool types.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 44, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of snapshot sizes in TiFlash-Proxy's Raft replication process. Critical for identifying abnormally large snapshots that could cause memory pressure, network congestion, or slow replica synchronization during TiFlash node recovery or scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1877, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy's future pool, broken down by pool name. Helps identify task queue buildup and potential thread pool saturation issues that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1395, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy worker queues, broken down by worker name. Essential for identifying task queue bottlenecks and worker thread saturation that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 6, "dashboard": "DM-Professional-Clinic", "title": "binlog pos", "group": "relay log", "used_for": "Used for monitoring the binlog position tracking in DM's relay log processing, comparing the master's binlog position against the relay log's current position. Essential for identifying replication lag and detecting if relay log processing is falling behind the upstream MySQL master.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 90, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring the rate of write commands and block operations per TiFlash instance. Helps identify write throughput imbalances across instances and diagnose storage write stall issues by tracking command execution patterns and block write activity.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "DM-Professional-Clinic", "title": "binlog file index", "group": "relay log", "used_for": "Used for tracking the current binlog file index position in DM's relay log processing, comparing the master's binlog file against the relay log's file position. Essential for monitoring replication lag and identifying binlog file synchronization delays between upstream MySQL and DM relay logs.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 2108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of threads in TiFlash-Proxy instances, showing how many threads are in different states (running, idle, blocked, etc.) per instance. Essential for diagnosing thread pool saturation, identifying I/O bottlenecks, and detecting thread starvation issues that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Throughput", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage layer throughput, separating write/ingest operations from delta management operations (compaction, GC). Critical for diagnosing write stalls by identifying whether bottlenecks stem from incoming writes or background maintenance tasks competing for I/O resources.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[2m]))", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 89, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Throughput By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring write and ingest throughput (bytes per second) for each TiFlash instance. Essential for identifying write performance bottlenecks, uneven write distribution across instances, and diagnosing storage write stall issues in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write\"}[2m])) by (instance)", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "DM-Professional-Clinic", "title": "write relay log duration", "group": "relay log", "used_for": "Used for monitoring the write latency of relay logs in DM (Data Migration) at different percentiles (p90, p95, p99). Critical for diagnosing relay log write performance issues and identifying bottlenecks in the data replication pipeline that could impact migration throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 238, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Mark Cache and Minmax Index Cache Memory Usage", "group": "Storage", "used_for": "Used for monitoring memory consumption of TiFlash's data indexing structures, including mark cache (data location index), minmax index files, and delta index cache. Essential for diagnosing TiFlash memory pressure issues and optimizing cache configurations when analytical query performance degrades.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_RNDeltaIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2258, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes per second) across different TiFlash-Proxy threads, filtered to show only threads with I/O rates exceeding 1KB/s. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or imbalanced workload distribution in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2660, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring voluntary context switches per second in TiFlash-Proxy threads, alerting when the rate exceeds 200 switches/second. High voluntary context switch rates typically indicate threads are frequently waiting for I/O operations or locks, helping diagnose I/O bottlenecks and thread contention issues in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 200" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "relay log", "used_for": "Used for monitoring the latency of reading binlog events from the relay log in DM (Data Migration), showing p90, p95, and p99 percentiles. Critical for diagnosing slow binlog processing issues and identifying performance bottlenecks in the data replication pipeline from upstream MySQL to TiDB.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 2661, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches in TiFlash-Proxy threads, which occur when the OS forcibly preempts threads due to resource contention or time slice expiration. High rates (>50/sec) indicate CPU saturation, thread contention, or I/O blocking issues that can degrade TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 50" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 137, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Total", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage I/O throughput, separating write/ingest operations from delta management operations (compaction, GC). Essential for diagnosing write stall issues by identifying whether bottlenecks stem from incoming writes or background delta management tasks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})", "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 128, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Disk Usage", "group": "PageStorage", "used_for": "Used for monitoring disk space usage and efficiency of TiFlash's PageStorage blob storage layer. Helps identify disk space issues, track valid data ratio, and diagnose storage bloat or inefficient space utilization that may require garbage collection or compaction.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})) by (instance)", "tiflash_system_asynchronous_metric_LogDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 83, "dashboard": "DM-Professional-Clinic", "title": "number of workers in different state", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of DM (Data Migration) workers across different operational states in the cluster. Essential for diagnosing high availability issues, detecting worker failures, and verifying proper worker state transitions during failover scenarios.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "count_values(\"state\", dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 86, "dashboard": "DM-Professional-Clinic", "title": "number of dm-masters start leader components per minute", "group": "HA", "used_for": "Used for monitoring the frequency of DM-master leader component initialization events per minute. Helps diagnose high availability issues, unexpected leader switches, or cluster instability when the counter shows frequent leader component restarts.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_master_start_leader_counter" ], "dashboard_file": "dm.json" }, { "panel_id": 25, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "relay log", "used_for": "Used for monitoring the size distribution of binlog events written to relay logs in DM (Data Migration), showing p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that could impact relay log performance and diagnose replication bottlenecks caused by oversized transactions.", "component": "", "phenomenon": "", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 68, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read Index Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's Read Index worker threads, which handle Raft read index requests to ensure read consistency. Essential for diagnosing performance bottlenecks when TiFlash queries experience high latency due to read index processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 129, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage File Num", "group": "PageStorage", "used_for": "Used for monitoring the number of blob files and log files in TiFlash's PageStorage engine per instance. Essential for tracking storage file accumulation and identifying potential issues with file compaction or cleanup processes that could impact storage efficiency and performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_BlobFileNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_asynchronous_metric_LogNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 138, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the efficiency of RocksDB get operations in TiFlash-Proxy, tracking hit rates across different storage layers including memtable, block cache (data/filter), and L0 SST files. Essential for diagnosing read performance issues and understanding how effectively the storage engine's multi-level cache hierarchy is serving get requests.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 81, "dashboard": "DM-Professional-Clinic", "title": "workers' state", "group": "HA", "used_for": "Used for monitoring the operational state of DM (Data Migration) workers in the cluster. Essential for tracking worker availability and diagnosing high availability issues when workers fail or become unavailable during data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (worker) (dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 125, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiFlash-Proxy, tracking maximum, 99th, and 95th percentile durations. Essential for diagnosing read performance issues and identifying slow data lookups in TiFlash's storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 123, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Number of Tables", "group": "PageStorage", "used_for": "Used for monitoring the number of tables stored in TiFlash's PageStorage across different storage pool modes (V2, V3, and mixed mode). Essential for tracking TiFlash data distribution and identifying storage mode transitions or table replication issues across instances.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 82, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB point lookup (Get operation) latency in TiFlash-Proxy, showing maximum, P99, and P95 durations in microseconds. Essential for diagnosing read performance issues and identifying slow Get operations that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 164, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Numer of Pages", "group": "PageStorage", "used_for": "Used for monitoring the number of pages currently held in memory by TiFlash's PageStorage engine. Essential for tracking memory usage patterns and diagnosing potential memory pressure or page cache issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_PagesInMem{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 71, "dashboard": "DM-Professional-Clinic", "title": "before any operate error", "group": "operate error", "used_for": "Used for monitoring errors that occur in DM (Data Migration) workers before any operation begins. Helps diagnose pre-operation validation failures, connection issues, or configuration problems that prevent DM tasks from starting properly.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"BeforeAnyOp\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 232, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PS Command OPS By Instance", "group": "PageStorage", "used_for": "Used for monitoring the operations per second (OPS) of PageStorage commands in TiFlash, broken down by instance and command type. Essential for tracking TiFlash storage layer activity and identifying instances with abnormal write/read command patterns or performance bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_page_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 133, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage WriteBatch Size", "group": "PageStorage", "used_for": "Used for monitoring the distribution of write batch sizes in TiFlash's PageStorage v3 engine. Helps diagnose write efficiency and identify whether data is being written in optimal batch sizes, which is critical for understanding I/O patterns and potential performance bottlenecks in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_page_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"v3\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "DM-Professional-Clinic", "title": "start error", "group": "operate error", "used_for": "Used for monitoring start operation errors in DM (Data Migration) workers. Essential for diagnosing failures when DM workers attempt to start data migration tasks or subtasks, helping identify configuration issues or resource constraints preventing successful task initialization.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Start\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 84, "dashboard": "DM-Professional-Clinic", "title": "number of pending shard ddl", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the count of pending DDL operations in DM (Data Migration) shard merge scenarios, categorized by task and type. Critical for identifying DDL coordination bottlenecks and ensuring smooth schema synchronization across sharded MySQL sources during high availability operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (dm_master_ddl_state_number)" ], "dashboard_file": "dm.json" }, { "panel_id": 163, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Tasks OPM", "group": "PageStorage", "used_for": "Used for monitoring the rate of garbage collection tasks in TiFlash's PageStorage engine, broken down by GC type. Helps identify PageStorage cleanup activity patterns and diagnose potential issues with storage space reclamation or excessive GC overhead impacting TiFlash performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 74, "dashboard": "DM-Professional-Clinic", "title": "pause error", "group": "operate error", "used_for": "Used for monitoring pause operation failures in DM (Data Migration) workers. Helps diagnose issues when attempting to pause data migration tasks, indicating potential control plane problems or worker communication failures.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Pause\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 85, "dashboard": "DM-Professional-Clinic", "title": "number of worker event error", "group": "HA", "used_for": "Used for monitoring the count of worker event errors in DM (Data Migration) Master, categorized by error type. Critical for identifying high availability issues and diagnosing worker coordination failures that could impact data migration reliability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (type) (dm_master_worker_event_error)" ], "dashboard_file": "dm.json" }, { "panel_id": 82, "dashboard": "DM-Professional-Clinic", "title": "shard ddl error per minute", "group": "HA", "used_for": "Used for monitoring the rate of shard DDL errors per minute in DM (Data Migration) master, broken down by task and error type. Critical for identifying DDL synchronization failures in sharded table migration scenarios and troubleshooting high availability issues in DM clusters.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (increase(dm_master_shard_ddl_error[1m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 129, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring RocksDB seek operation rates in TiFlash-Proxy, including total seeks, successful seeks, and next operations. Essential for diagnosing read performance issues and understanding data access patterns in TiFlash's storage layer, particularly when investigating slow queries or inefficient range scans.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 126, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write operation latency in TiFlash-Proxy's RocksDB storage engine, showing maximum, 99th percentile, and 95th percentile write durations. Essential for diagnosing TiFlash write performance bottlenecks and identifying when storage layer operations are causing delays in data synchronization from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 72, "dashboard": "DM-Professional-Clinic", "title": "source bound error", "group": "operate error", "used_for": "Used for monitoring source binding operation errors in DM (Data Migration) workers. Helps diagnose connection or configuration issues when DM workers fail to bind to upstream MySQL/MariaDB data sources, which directly impacts data replication throughput.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"SourceBound\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 128, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiFlash-Proxy, broken down by operation type. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification patterns that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 167, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Stale Read OPS", "group": "Raft", "used_for": "Used for monitoring the rate of stale read operations per second on each TiFlash instance. Helps track the usage of stale read feature and identify instances handling high volumes of stale read requests, which can indicate read-heavy workloads tolerating slightly outdated data for better performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_stale_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 135, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) file synchronization operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing write performance bottlenecks and disk I/O issues, as high WAL sync duration directly impacts transaction commit latency and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 231, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Pending Writers Num", "group": "PageStorage", "used_for": "Used for monitoring the number of pending write operations in TiFlash's PageStorage component. Helps identify write bottlenecks and I/O congestion when pending writers accumulate, indicating potential disk performance issues or write pressure on TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_PSPendingWriterNum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 137, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations in TiFlash-Proxy's RocksDB engine. Critical for diagnosing write durability overhead and identifying potential I/O bottlenecks when WAL sync frequency becomes excessive or irregular.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 78, "dashboard": "DM-Professional-Clinic", "title": "resume error", "group": "operate error", "used_for": "Used for monitoring the error rate of resume operations in DM (Data Migration) workers. Critical for identifying failures when attempting to resume paused or failed data migration tasks, helping diagnose task recovery issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Resume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 136, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiFlash-Proxy, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage performance issues, as prolonged compaction times can indicate write amplification problems or resource bottlenecks affecting query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 69, "dashboard": "DM-Professional-Clinic", "title": "auto-resume error", "group": "operate error", "used_for": "Used for monitoring auto-resume operation failures in DM (Data Migration) workers. Critical for identifying when automatic task recovery mechanisms fail, requiring manual intervention to restore data synchronization pipelines.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"AutoResume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 35, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Read Index OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations per second in TiFlash instances. Helps identify read throughput patterns and diagnose issues when TiFlash queries require strong consistency reads through the Raft protocol.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 130, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) operations in TiFlash-Proxy's RocksDB storage engine, tracking maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues that affect data durability and write throughput in TiFlash.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 242, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Heavy Raft Apply Duration", "group": "Raft", "used_for": "Used for monitoring the 99th percentile latency of Raft command apply operations in TiFlash. Critical for diagnosing replication lag and data synchronization performance issues between TiKV and TiFlash, especially when queries on TiFlash show stale data or high latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 75, "dashboard": "DM-Professional-Clinic", "title": "stop error", "group": "operate error", "used_for": "Used for monitoring the frequency of stop operation errors in DM (Data Migration) workers. Critical for identifying issues when attempting to stop data migration tasks, helping diagnose worker management and task lifecycle problems.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Stop\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 140, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file read operations in TiFlash-Proxy's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage performance issues and identifying slow disk I/O or file system bottlenecks that impact analytical query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 103, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables in TiFlash-Proxy, broken down by column family. Essential for diagnosing memory pressure issues and understanding write buffer usage that can impact write performance and memory stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 120, "dashboard": "DM-Professional-Clinic", "title": "Validator Error Count", "group": "Validator Statistics", "used_for": "Used for monitoring the total number of validation errors encountered by DM's data validator component. Essential for identifying data consistency issues during migration and quickly detecting when source and target data diverge or validation checks fail.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_error_count" ], "dashboard_file": "dm.json" }, { "panel_id": 162, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Duration", "group": "PageStorage", "used_for": "Used for monitoring the 99th percentile latency of TiFlash PageStorage garbage collection operations. Critical for diagnosing TiFlash storage performance issues and identifying when GC processes are taking too long, which can impact query performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 77, "dashboard": "DM-Professional-Clinic", "title": "update error", "group": "operate error", "used_for": "Used for monitoring the rate of update operation errors in DM (Data Migration) workers. Helps identify issues with metadata updates or configuration changes that may be failing, indicating potential problems with DM's operational stability or connectivity to PD.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Update\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 116, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the data replication lag in bytes between the DM validator and the source binlog position. Critical for identifying data synchronization delays and ensuring the validator is keeping pace with upstream changes during data migration validation.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logpos_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 82, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration", "group": "Raft", "used_for": "Used for monitoring the p99 latency of applying Raft write logs in TiFlash's storage engine. Critical for diagnosing write performance bottlenecks and replication lag issues when TiFlash processes data changes from TiKV through the Raft protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m]))", "sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m]) ) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (decode)", "group": "Raft", "used_for": "Used for monitoring the time spent decoding Raft log data during region writes in TiFlash. Critical for diagnosing write path performance issues, particularly when data deserialization becomes a bottleneck affecting TiFlash's ability to replicate data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 88, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the memtable hit rate in TiFlash-Proxy's RocksDB storage engine. Helps diagnose memory efficiency and cache effectiveness by showing how often data is found in memory versus requiring disk reads, which is critical for identifying performance bottlenecks in TiFlash query processing.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 37, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Wait Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash waiting for Raft log index synchronization at p80, p95, and p99 percentiles. Critical for diagnosing data replication delays between TiKV and TiFlash, helping identify when TiFlash replicas fall behind in applying Raft logs, which can impact query freshness and read performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 36, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Batch Read Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of batch read index operations in TiFlash's Raft layer at different percentiles (p80, p95, p99). Critical for diagnosing TiFlash read performance issues and identifying delays in obtaining consistent read snapshots from the Raft consensus protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiFlash-Proxy's RocksDB engine, tracking maximum, P99, and P95 latencies when writes are temporarily blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when TiFlash storage layer experiences backpressure that could impact data ingestion from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 158, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page write Duration", "group": "PageStorage", "used_for": "Used for monitoring write latency percentiles (P80, P95, P99, P999) in TiFlash's PageStorage engine. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow page write operations that could impact columnar data ingestion and storage efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 102, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the top 20 block cache sizes across different column families in TiFlash-Proxy's RocksDB engine. Essential for diagnosing memory usage patterns and identifying potential cache pressure issues that could impact read performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 114, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog file position being processed by the DM (Data Migration) validator component. Helps track data validation progress and identify potential lag or stalls in the validation process during MySQL-to-TiDB migration.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_file" ], "dashboard_file": "dm.json" }, { "panel_id": 198, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage stored bytes by type", "group": "PageStorage", "used_for": "Used for monitoring the distribution of stored data bytes across different types in TiFlash's PageStorage engine. Helps identify storage space usage patterns and diagnose potential storage imbalances or capacity issues by data type.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_page_data_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (write blocks)", "group": "Raft", "used_for": "Used for monitoring the latency distribution of TiFlash region write operations when persisting data blocks to storage. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow storage I/O issues that impact data replication from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 467, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write traffic flow and data insertion rate of TiFlash-Proxy's RocksDB block cache. Essential for analyzing cache utilization patterns and diagnosing memory pressure or cache efficiency issues in TiFlash storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 81, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft write log commands in TiFlash nodes. Essential for diagnosing write replication performance issues and identifying when TiFlash struggles to keep up with Raft log application from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 112, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog position (in bytes) being processed by the DM validator component. Helps track data validation progress and identify potential lag or stalls in the validation process during data migration tasks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_pos" ], "dashboard_file": "dm.json" }, { "panel_id": 80, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the block cache hit ratio in TiFlash-Proxy's RocksDB storage engine, including overall cache hits, data block hits, and filter block hits. Essential for diagnosing read performance issues and determining if the block cache size is sufficient to avoid excessive disk I/O.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring TiProxy server CPU utilization and comparing it against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation, and determining if TiProxy instances need vertical scaling or load redistribution.", "component": "TIPROXY", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}[2m])", "tiproxy_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 118, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the latency of validator binlog processing from file in DM (Data Migration). Helps diagnose delays in data validation when reading and processing binlog events from local files, which is critical for identifying bottlenecks in the validation pipeline.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logfile_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 235, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft consensus events per second in TiFlash nodes, broken down by event type. Essential for diagnosing Raft replication issues, identifying abnormal consensus activity, and troubleshooting data synchronization problems between TiFlash replicas.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 131, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys in each column family of TiFlash-Proxy's RocksDB storage engine. Helps assess data volume distribution across column families and identify potential storage imbalances or unexpected data growth in TiFlash's storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 468, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiFlash-Proxy's RocksDB storage engine, including overall blocks, data blocks, and filter blocks. Essential for understanding cache population patterns and diagnosing issues with cache efficiency or excessive cache churn that could impact read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 132, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiFlash-Proxy's RocksDB storage engine, including keys read, keys written, and corrupt keys detected during compaction. Essential for diagnosing data flow issues, identifying unusual read/write patterns, and detecting potential data corruption problems in TiFlash storage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiProxy's memory consumption, including both resident memory (RSS) and Go heap memory usage. Essential for detecting memory leaks, identifying out-of-memory risks, and understanding TiProxy's resource footprint during connection pooling and query routing operations.", "component": "TIPROXY", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 105, "dashboard": "DM-Professional-Clinic", "title": "replication transaction batch", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile batch size of replicated transactions in DM's binlog replication process. Helps identify transaction batching efficiency and diagnose replication throughput issues when batch sizes are unexpectedly small or large.", "component": "", "phenomenon": "io", "object": "transaction", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_transaction_batch_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task,queueNo,type))" ], "dashboard_file": "dm.json" }, { "panel_id": 6, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiProxy instances. Essential for tracking connection pool utilization, identifying connection leaks, and diagnosing capacity issues when connection counts approach limits or show unexpected spikes.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 241, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Frequent Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of frequent Raft protocol events in TiFlash, broken down by event type. Essential for diagnosing Raft replication activity, identifying abnormal event patterns, and troubleshooting data synchronization issues between TiFlash and TiKV.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_frequent_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Create Connection OPM", "group": "Server", "used_for": "Used for monitoring the rate of new connection creation per minute on TiProxy instances. Helps identify connection churn patterns, sudden spikes in client connection attempts, and potential connection pool issues or application reconnection storms.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_create_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 5, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiProxy instance has been running since its last restart. Essential for tracking instance stability, identifying unexpected restarts, and verifying deployment or maintenance activities.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 93, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft admin logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft administrative commands in TiFlash through a heatmap visualization. Essential for diagnosing performance issues related to Raft admin operations like region splits, merges, and configuration changes that can impact TiFlash's data replication and cluster coordination.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 675, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations on the lock column family within TiFlash-Proxy's scheduler for specific commands. Helps diagnose lock-related scanning performance and identify potential lock contention or scanning bottlenecks in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 243, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Entry Batch Size Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log entry batch sizes in TiFlash as a heatmap. Helps identify abnormal Raft replication patterns, such as unusually large or small batch sizes that could indicate network issues, replication bottlenecks, or configuration problems affecting data synchronization performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_entry_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"normal\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 234, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Log Gap Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft log gaps between applied_index and compact_index in TiFlash replicas. Helps identify replication lag issues and diagnose when TiFlash is falling behind in applying or compacting Raft logs, which can impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"applied_index\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"compact_index\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 133, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data read per RocksDB read operation in TiFlash-Proxy, showing maximum, P99, and P95 values. Helps diagnose read amplification issues and identify whether queries are reading excessive data, which can indicate inefficient storage layout or suboptimal query patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 139, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiFlash-Proxy's RocksDB storage engine, including successful writes (self/other), timeouts, and WAL writes. Essential for diagnosing write performance bottlenecks, identifying write timeout issues, and understanding write-ahead logging activity in TiFlash storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiProxy instances. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact proxy performance and stability.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiproxy\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 127, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the volume of data pending compaction in TiFlash-Proxy's RocksDB storage engine, broken down by column family. Critical for identifying compaction backlogs that can lead to read amplification, increased disk usage, and degraded I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 85, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of TiFlash-Proxy's RocksDB storage engine, tracking both direct bytes read and iterator-based reads. Essential for diagnosing TiFlash read performance issues, identifying I/O bottlenecks, and understanding data access patterns during analytical query execution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 10, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Owner", "group": "Server", "used_for": "Used for identifying which TiProxy instance is currently serving as the owner/leader in the cluster. Essential for troubleshooting leadership changes, failover events, and understanding which instance is actively managing cluster coordination.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_owner{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 8, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Disconnection OPM", "group": "Server", "used_for": "Used for monitoring the rate of client disconnections per minute in TiProxy, categorized by disconnection type. Essential for identifying connection stability issues, detecting abnormal disconnect patterns, and diagnosing network or client-side problems affecting database connectivity.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 249, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ongoing raft snapshot", "group": "Raft", "used_for": "Used for monitoring the rate of ongoing Raft snapshot data transfer in TiFlash, broken down by snapshot type (apply/generate). Essential for diagnosing region replication issues, identifying snapshot bottlenecks, and understanding data synchronization overhead between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 86, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring write throughput in TiFlash-Proxy's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written. Essential for diagnosing write performance bottlenecks, identifying I/O saturation issues, and understanding data ingestion rates in TiFlash's storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 90, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the data flow rates during RocksDB compaction operations in TiFlash-Proxy, including bytes read, bytes written during compaction, and flush write bytes. Essential for diagnosing I/O bottlenecks, understanding storage engine write amplification, and identifying excessive compaction activity that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 240, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft eager garbage collection operations in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential performance impacts from excessive eager cleanup operations in the Raft layer.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_eager_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 13, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Instance", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency for each TiProxy instance. Essential for identifying performance outliers and diagnosing which specific TiProxy instances are experiencing elevated query latency that could impact client connections.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 239, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC Duration", "group": "Raft", "used_for": "Used for monitoring the latency of Raft eager garbage collection operations in TiFlash at p95 and p99 percentiles. Critical for diagnosing performance issues related to TiFlash's Raft log cleanup process, which can impact write throughput and storage efficiency when GC operations become slow.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_eager_gc_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_raft_eager_gc_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency through TiProxy at the 95th and 99th percentiles. Essential for identifying performance degradation in client-to-TiDB connections and diagnosing whether slowdowns originate from the proxy layer or backend TiDB servers.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 72, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding Raft snapshots during data replication. Critical for diagnosing slow snapshot application issues and identifying bottlenecks in TiFlash's Raft log replay process, especially during region transfers or cluster scaling operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 244, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of region sizes in TiFlash, showing both flushed and unflushed data volumes across different size buckets. Helps identify region size imbalances, detect abnormally large regions that may impact I/O performance, and monitor the effectiveness of TiFlash's data flushing mechanism.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"unflushed\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"flushed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 518, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiFlash-Proxy's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O and degrade read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 516, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiFlash-Proxy's storage engine. Helps identify potential memory leaks or snapshot accumulation issues that could impact TiFlash-Proxy performance and resource usage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 863, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression efficiency across different LSM-tree levels in TiFlash-Proxy's RocksDB storage engine. Helps identify compression effectiveness and potential storage optimization opportunities, with lower ratios indicating better compression and reduced disk space usage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Backend", "group": "Query Summary", "used_for": "Used for monitoring the connection rate (connections per second) distributed across different TiDB backend instances through TiProxy. Helps identify backend load distribution imbalances and diagnose connection routing issues when certain backends receive disproportionate traffic.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 2381, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency of stall condition changes in TiFlash-Proxy's RocksDB column families. Helps diagnose write throttling and backpressure issues when RocksDB triggers stalls due to memtable limits, L0 file accumulation, or pending compaction bytes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 15, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Instance", "group": "Query Summary", "used_for": "Used for monitoring the number of commands per second (CPS) processed by each TiProxy instance. Helps identify load distribution across TiProxy instances and detect performance bottlenecks or imbalanced traffic routing.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 250, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft snapshot sizes in TiFlash over time as a heatmap. Helps identify abnormal snapshot size patterns, detect large snapshot transfers that may impact network bandwidth, and diagnose replication performance issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_snapshot_total_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"approx_raft_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2003, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST file ingestion operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing snapshot application performance issues and identifying bottlenecks during data replication from TiKV to TiFlash.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 74, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ingest SST Duration", "group": "Raft", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations in TiFlash's Raft layer. Critical for diagnosing performance issues during data replication from TiKV to TiFlash, especially when experiencing slow data synchronization or snapshot application delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 73, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Flush Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash takes to flush Raft snapshots to disk. Critical for diagnosing slow snapshot application issues that can cause TiFlash replicas to lag behind TiKV, impacting data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) in TiProxy, broken down by command type (SELECT, INSERT, UPDATE, etc.). Helps identify query workload distribution and detect unusual command patterns or sudden changes in specific query types.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cmd_type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 4453, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiFlash-Proxy's encryption key management system. Essential for tracking encryption key lifecycle and diagnosing issues with data-at-rest encryption configuration or key rotation problems.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 127, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode SST to DT Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding SST files to DeltaTree format during Raft snapshot application. Critical for diagnosing slow snapshot ingestion and identifying performance bottlenecks when TiFlash replicas are catching up with data changes from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode_sst2dt\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 14, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Backend", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency from TiProxy to each backend TiDB instance. Essential for identifying which specific backend servers are experiencing slow query performance and diagnosing uneven load distribution or backend-specific issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, backend)), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 71, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Keys flow", "group": "Raft", "used_for": "Used for monitoring the rate of Raft log keys processed by TiFlash, categorized by operation type (such as apply, snapshot). Essential for tracking data replication throughput from TiKV to TiFlash and diagnosing replication lag or bottlenecks in the Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 75, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Applying snapshots Count", "group": "Raft", "used_for": "Used for monitoring the number of Raft snapshots currently being applied and prehandled in TiFlash instances. Critical for diagnosing region replication delays, identifying snapshot application bottlenecks, and troubleshooting data synchronization issues between TiKV and TiFlash during region transfers or cluster scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumPrehandlingSubTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 134, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiFlash-Proxy's RocksDB storage engine. Helps diagnose write amplification issues and identify abnormally large write operations that could impact I/O performance and storage efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4492, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiFlash-Proxy's storage layer. Helps verify encryption-at-rest is functioning correctly and track the volume of encrypted data files across the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 92, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log replication latency from TiKV to TiFlash instances over time. Essential for diagnosing data synchronization delays and identifying network or replication bottlenecks between TiKV and TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2452, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons causing write stalls in TiFlash-Proxy's RocksDB storage engine. Essential for diagnosing write performance degradation by identifying specific bottlenecks such as memtable limits, L0 file accumulation, or pending compaction issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2002, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in TiFlash-Proxy's RocksDB storage engine for each column family. Essential for diagnosing compaction issues, storage amplification problems, and identifying when LSM-tree structure becomes unbalanced due to write pressure or compaction bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 26, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Ping Backend Duration", "group": "Backend", "used_for": "Used for monitoring the network latency between TiProxy and backend TiDB servers through health check pings. Essential for diagnosing connection quality issues and identifying network problems that could affect query routing performance.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_backend_ping_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 21, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration OPM", "group": "Balance", "used_for": "Used for monitoring the rate of session migrations per minute in TiProxy's load balancing system, showing migration flows between backend TiDB instances and their success/failure status. Essential for diagnosing connection rebalancing behavior, identifying migration bottlenecks, and troubleshooting session redistribution issues during scaling or failover events.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(label_replace(sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (migrate_res, from, to), \"from\", \"$1\", \"from\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\"), \"to\", \"$1\", \"to\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 23, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Reasons", "group": "Balance", "used_for": "Used for tracking the reasons why TiProxy migrates client sessions between TiDB instances during load balancing operations. Essential for understanding session migration patterns and diagnosing whether migrations are triggered by planned rebalancing, backend failures, or configuration changes.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 99, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the effectiveness of TiFlash's rough set filter in eliminating unnecessary data reads during query execution. Helps identify whether the rough set index is efficiently filtering out data blocks, which is critical for diagnosing slow analytical query performance and optimizing TiFlash storage efficiency.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (instance)", "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2451, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of RocksDB compaction triggers in TiFlash-Proxy by column family and reason type. Helps diagnose storage layer performance issues by identifying whether compactions are triggered by size thresholds, file counts, or manual operations, which is critical for understanding write amplification and I/O patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 20, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Backend Connections", "group": "Balance", "used_for": "Used for monitoring the number of active backend connections from TiProxy to TiDB instances. Essential for tracking connection distribution across TiDB nodes and diagnosing connection balancing issues or connection pool saturation.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_balance_b_conn{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 4496, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether encryption-at-rest has been successfully initialized in TiFlash-Proxy instances. Essential for confirming encryption feature activation during cluster setup or troubleshooting encryption-related configuration issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Handshake Duration", "group": "Query Summary", "used_for": "Used for monitoring the 95th and 99th percentile latency of client connection handshakes through TiProxy. Critical for diagnosing slow connection establishment issues and identifying authentication or SSL/TLS negotiation bottlenecks that impact client connection times.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_handshake_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_handshake_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 517, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the age of the oldest active snapshot in TiFlash-Proxy's RocksDB engine. Critical for identifying long-running transactions or stale snapshots that can block compaction, cause storage bloat, and degrade read/write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 97, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate Histogram", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the distribution of rough set filter effectiveness in TiFlash's storage layer. Helps identify how efficiently TiFlash is filtering data blocks during queries, with higher filter rates indicating better query performance through reduced data scanning.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 91, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency", "group": "Raft", "used_for": "Used for monitoring the latency of Raft log replication from TiKV to TiFlash at p95, p99, and maximum percentiles. Critical for diagnosing data synchronization delays between TiKV and TiFlash, helping identify network issues or upstream bottlenecks that affect TiFlash's real-time data availability.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 25, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Get Backend Duration", "group": "Backend", "used_for": "Used for monitoring the latency of TiProxy selecting and connecting to backend TiDB servers, showing p95, p99, and average durations. Critical for diagnosing connection establishment delays and backend selection performance issues that could impact client query response times.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_backend_get_backend_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_backend_get_backend_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 4497, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the size of encryption metadata files in TiFlash-Proxy. Helps track storage overhead from encryption key management and identify potential issues with metadata file growth that could impact disk space or I/O performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 22, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Duration", "group": "Balance", "used_for": "Used for monitoring the latency of session migration operations in TiProxy during load balancing, showing p95 and p99 percentiles. Critical for diagnosing performance issues when TiProxy redistributes client connections across TiDB instances, helping identify slow migrations that could impact user experience during rebalancing events.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_balance_migrate_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_balance_migrate_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 187, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload Duration", "group": "Disaggregated", "used_for": "Used for monitoring the latency of checkpoint upload operations in TiFlash disaggregated storage architecture at different percentiles (p80, p90, p99). Critical for diagnosing slow checkpoint synchronization issues that could impact data durability and recovery time in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 27, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Health Check Cycle", "group": "Backend", "used_for": "Used for monitoring the health check cycle duration of backend TiDB servers in TiProxy. Essential for diagnosing connection pool health and detecting delays in backend availability detection that could impact connection routing and failover responsiveness.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_backend_health_check_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 4498, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the latency of reading and writing encryption metadata files in TiFlash-Proxy, showing p95 and maximum durations. Critical for diagnosing performance issues with encryption-at-rest operations and identifying when metadata I/O becomes a bottleneck affecting TiFlash query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 173, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency breakdown of read operations in TiFlash's disaggregated storage architecture. Helps identify which component (compute or storage layer) is contributing most to read latency, essential for diagnosing performance bottlenecks in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 29, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of data received by TiProxy from backend TiDB servers, measured in bytes per second. Helps identify backend traffic patterns, detect imbalanced data distribution across TiDB instances, and diagnose potential network bottlenecks between TiProxy and backend servers.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 196, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload keys speed by type (all)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint key uploads in TiFlash's disaggregated storage architecture, broken down by key type. Helps diagnose checkpoint performance issues and identify which key types are contributing most to checkpoint upload activity during data persistence operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_keys_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 30, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets received by TiProxy from backend TiDB servers. Helps identify network throughput patterns, backend communication issues, and potential packet loss or congestion between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 53, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the storage space consumed by each TiFlash instance, including both local and remote storage usage. Essential for tracking storage capacity trends, planning disk space allocation, and identifying instances approaching storage limits before they impact cluster operations.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"\"}) by (instance)", "sum(tiflash_system_current_metric_StoreSizeUsedRemote{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 6, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiDB server memory consumption, including total resident memory and Go heap allocation. Essential for detecting memory leaks, identifying out-of-memory risks, and optimizing memory configuration when TiDB instances experience high memory pressure or OOM issues.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 54, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Available size", "group": "Server", "used_for": "Used for monitoring the available disk space on each TiFlash instance. Critical for preventing storage exhaustion issues and planning capacity expansion before TiFlash nodes run out of disk space.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 33, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Cross Location Bytes/Second", "group": "Traffic", "used_for": "Used for monitoring the rate of cross-location network traffic in bytes per second handled by TiProxy instances. Essential for identifying cross-region data transfer costs and diagnosing network bottlenecks when TiDB clusters span multiple availability zones or geographic locations.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_traffic_cross_location_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 55, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Capacity size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity allocated to each TiFlash instance. Essential for capacity planning and identifying instances approaching storage limits before they impact cluster availability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 174, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow", "group": "Disaggregated", "used_for": "Used for monitoring the data upload flow rate to remote storage in TiFlash disaggregated architecture, tracking both incremental checkpoint uploads and compaction-related uploads. Essential for diagnosing write-to-S3 performance issues and understanding checkpoint synchronization patterns in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"incremental\"}[2m]))", "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"compaction\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 197, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow by type (incremental+compaction)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint data upload flow in TiFlash's disaggregated storage architecture, broken down by operation type (incremental updates and compaction). Essential for diagnosing checkpoint upload performance, identifying bottlenecks in data persistence, and understanding the balance between incremental and compaction workloads in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 175, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Store Usage", "group": "Disaggregated", "used_for": "Used for monitoring remote storage usage in disaggregated TiFlash architecture, tracking total size, valid data size, and storage efficiency ratio per instance. Essential for managing remote storage capacity, identifying storage bloat, and ensuring efficient data retention in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"}) by (instance)", "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) by (instance)", "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"})) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Status", "group": "Disaggregated", "used_for": "Used for monitoring the status of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for tracking S3-based storage cleanup activities and diagnosing issues with stale data removal in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_s3_gc_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance,type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 31, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the outbound network traffic rate from TiProxy instances to backend TiDB servers. Helps identify network bandwidth usage patterns, detect traffic imbalances across backends, and diagnose connectivity or throughput issues between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 9, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Failed Query OPS", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per second in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing issues like connection problems, syntax errors, or resource constraints affecting query success rates.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 11, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) executed by each TiDB instance. Helps identify load distribution imbalances across TiDB servers and pinpoint which instances are handling the most query traffic.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 8, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency at the 95th and 99th percentiles in TiDB, excluding internal queries. Essential for identifying performance degradation, detecting slow query patterns, and understanding tail latency that affects user experience during troubleshooting or capacity planning.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", sql_type!=\"internal\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 4, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances. Essential for tracking connection usage patterns, identifying connection leaks, and diagnosing issues related to connection limits or sudden connection spikes that may impact cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 5, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiDB server instances over a 2-minute window. Essential for identifying CPU bottlenecks, resource contention, and determining if TiDB servers need vertical scaling or if query workloads need optimization.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 15, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Parse Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL parsing latency in TiDB. Helps identify performance bottlenecks in the query parsing phase, particularly when dealing with complex SQL statements or syntax issues that cause excessive parsing overhead.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 178, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Bytes", "group": "S3", "used_for": "Used for monitoring S3 storage I/O throughput in TiFlash, tracking read/write bytes per second including DMFile operations. Essential for diagnosing S3 storage performance bottlenecks and understanding data transfer patterns when TiFlash uses S3 as the storage backend.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3WriteDMFileBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Execution Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of general SQL queries in TiDB. Essential for identifying query performance degradation and tail latency issues that affect user experience, helping diagnose slow query problems before they impact the majority of requests.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 200, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region", "group": "Server", "used_for": "Used for monitoring the total number of Raft regions stored on each TiFlash instance. Essential for tracking data distribution across TiFlash nodes and identifying region imbalance issues that could impact query performance and storage utilization.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 21, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiFlash instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and verify cluster continuity after maintenance or unexpected failures.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 181, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "IO Throughput", "group": "Server", "used_for": "Used for monitoring the I/O throughput rate (bytes per second) across TiFlash instances. Helps identify I/O bottlenecks, uneven load distribution, and storage performance issues that may impact analytical query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 193, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the P90 and P99 latency breakdown of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for diagnosing performance bottlenecks in S3-based storage cleanup processes and identifying which GC operation types are experiencing high latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 236, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexCache", "group": "Disaggregated", "used_for": "Used for monitoring the delta index cache hit rate and access patterns in TiFlash's disaggregated storage architecture. Essential for evaluating cache efficiency and diagnosing performance issues related to delta index lookups, where low hit rates may indicate insufficient cache size or suboptimal data access patterns.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, instance)", "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit\"}[2m])) by (instance) /sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 51, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiFlash instances, showing both actual CPU usage rate and total available logical CPU cores. Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiFlash nodes need vertical or horizontal scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])", "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 189, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Request QPS", "group": "Disaggregated", "used_for": "Used for monitoring the rate of remote object lock requests per second in TiFlash's disaggregated storage architecture, broken down by request type. Essential for diagnosing lock contention issues and understanding coordination overhead between compute and storage nodes in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_disaggregated_object_lock_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 13, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) in TiDB, broken down by command type (SELECT, INSERT, UPDATE, DELETE, etc.). Helps identify query workload composition and detect unusual command patterns or sudden changes in specific query types.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 191, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Duration", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency of remote object lock requests in TiFlash's disaggregated storage architecture. Critical for diagnosing performance bottlenecks in distributed object storage access and identifying lock contention issues that may impact query execution in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (success/error). Essential for tracking overall query throughput, comparing current performance against historical baselines (1-day offset), and quickly identifying sudden drops in successful queries or spikes in failed commands.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 179, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operations (PUT, GET, HEAD) performed by TiFlash instances when using S3-compatible storage. Essential for diagnosing S3 storage throughput issues, identifying unusual access patterns, and troubleshooting performance problems related to remote storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "Used for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for tracking cluster workload patterns, identifying traffic spikes, and understanding the distribution of different query types during performance analysis or troubleshooting.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 201, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Unit", "group": "Disaggregated", "used_for": "Used for monitoring Request Unit (RU) consumption in TiFlash's disaggregated storage and compute architecture. Helps track resource usage patterns across keyspaces and compute clusters, essential for capacity planning and cost analysis in serverless TiDB deployments.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)", "sum(increase(tiflash_storage_sync_replica_ru[24h])) by (keyspace_id)", "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)", "sum(increase(tiflash_compute_request_unit[24h])) by (cluster_id)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 141, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Import Service", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST import service threads, which handle importing SST files during data replication and recovery. Essential for diagnosing performance bottlenecks during data ingestion, region migration, or when TiFlash replicas are being synchronized from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"sst_importer.*\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 32, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets sent from TiProxy instances to backend TiDB servers. Helps identify network traffic distribution patterns, detect connection issues to specific backends, and diagnose packet-level communication problems between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 10, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory", "group": "Server", "used_for": "Used for monitoring TiFlash memory allocation patterns through jemalloc metrics, including retained, mapped, and resident memory. Essential for diagnosing memory leaks, understanding memory pressure, and identifying when TiFlash instances are approaching OOM conditions.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_MemoryCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 18, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Query Detail", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache, broken down by cache type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 16, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Compile Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL compilation latency in TiDB, measuring the time spent parsing and optimizing queries before execution. Essential for diagnosing slow query performance caused by complex SQL statements or inefficient query plans, particularly when execution time is normal but overall latency is high.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 233, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory Usage of Storage Tasks", "group": "Disaggregated", "used_for": "Used for monitoring memory consumption of storage-related tasks in TiFlash's disaggregated architecture, including query storage tasks, page fetching operations, and delta index cache. Critical for diagnosing memory pressure issues and optimizing resource allocation in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DT_DeltaIndexCacheSize{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingSharedColumnData{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 182, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Retry OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operation retries in TiFlash, including GET, PUT, and DMFile operations. Essential for identifying S3 connectivity issues, network instability, or storage service degradation that causes TiFlash to retry operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFileRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 147, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash region worker threads, which handle region data synchronization from TiKV. Essential for diagnosing performance bottlenecks in TiFlash's data replication pipeline and identifying when region workers are CPU-saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "TPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring transactions per second (TPS) in TiDB, broken down by transaction type and mode (optimistic/pessimistic). Essential for tracking transaction throughput, identifying workload patterns, and detecting sudden changes in transaction volume that may indicate application issues or load shifts.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 22, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Statement Num", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the maximum number of SQL statements executed within a single transaction across TiDB instances. Critical for identifying transactions with excessive statement counts that may cause performance issues, lock contention, or transaction conflicts.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 21, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring transaction execution latency at different percentiles (p80, p95, p99) across TiDB instances. Essential for identifying transaction performance degradation, detecting slow transaction patterns, and diagnosing issues affecting overall database responsiveness during transactional workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 176, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote File Num", "group": "Disaggregated", "used_for": "Used for monitoring the number of remote storage files in TiFlash's disaggregated storage architecture. Helps track remote file accumulation and identify potential issues with file management or storage tiering in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"num_files\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 180, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Request Duration", "group": "S3", "used_for": "Used for monitoring S3 request latency in TiFlash at different percentiles (p80, p90, p99, p100). Critical for diagnosing performance issues when TiFlash uses S3-compatible object storage, helping identify slow S3 operations that may impact query performance or data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 154, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Apply", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST apply threads, which handle low-priority Raft log applications to the storage engine. Critical for diagnosing write performance bottlenecks and identifying when apply threads are saturated during heavy write workloads or data replication.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 185, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Operations", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache performance when using S3-compatible storage, tracking cache hit/miss rates and operation types. Essential for diagnosing I/O efficiency issues and optimizing cache configuration when TiFlash operates in disaggregated storage mode.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit|dtfile_miss\"}[2m]))", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit|page_miss\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and whether servers have been restarted during troubleshooting or maintenance operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 186, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Flow", "group": "S3", "used_for": "**Used For:**\n\nUsed for monitoring the rate of data flow through TiFlash's remote cache (S3-backed cache), broken down by operation type. Essential for diagnosing S3 cache performance, identifying cache hit/miss patterns, and troubleshooting remote storage access issues in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 155, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Store", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of Raft Store threads in TiFlash's proxy layer, which handle Raft consensus operations and data replication. Critical for diagnosing replication performance issues, identifying CPU bottlenecks in Raft processing, and detecting when Raft Store threads are saturated during heavy write workloads or region operations.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 251, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's apply worker threads, which handle Raft log application and data replication from TiKV. Critical for diagnosing replication lag issues and identifying CPU bottlenecks in TiFlash's data synchronization process.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 188, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Usage", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache utilization when using S3-based disaggregated storage, tracking both DTFile cache and page cache capacity versus actual usage. Essential for diagnosing cache efficiency issues, identifying cache pressure, and optimizing remote storage performance in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_DTFileCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DTFileCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 23, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the maximum number of retries for transactions in TiDB due to conflicts or lock timeouts. Critical for identifying transaction contention issues and determining if applications need retry logic optimization or if write conflicts are causing performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 145, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Task", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash region task threads, which handle region-related operations like Raft log application and region metadata management. Essential for diagnosing TiFlash performance issues related to region synchronization overhead and identifying instances with abnormal region processing workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"})" ], "dashboard_file": "tiflash-summary.json" } ]