[ { "panel_id": 448, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Queuing Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time backup/restore requests spend waiting in queue before being processed by TiKV's import engine during Point In Time Restore operations. Critical for identifying bottlenecks in restore performance and determining if the system is overloaded with concurrent restore requests.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 321, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing User Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of active user transactions in TiDB. Critical for identifying long-running transactions that may block garbage collection (GC) and cause performance degradation or storage bloat.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 203, "dashboard": "TiDB-Cluster-Clinic", "title": "No Available Connection Counter", "group": "Batch Client", "used_for": "Used for monitoring connection unavailability in TiDB's batch client when communicating with TiKV. Helps diagnose connection pool exhaustion or network connectivity issues that prevent TiDB from establishing connections to TiKV nodes.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 85, "dashboard": "TiDB-Cluster-Clinic", "title": "Worker Action OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) worker operations per minute in TiDB, broken down by action type. Essential for diagnosing GC performance issues, identifying bottlenecks in the cleanup process, and ensuring efficient reclamation of obsolete MVCC versions.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 449, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Throughput", "group": "Point In Time Restore", "used_for": "Used for monitoring the throughput of apply requests during Point In Time Restore (PITR) operations in TiKV, measured in bytes per second per instance. Essential for tracking restore performance and identifying bottlenecks when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 457, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Store Memory Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring memory consumption of the Raft Store component in TiKV instances during Point In Time Restore operations. Essential for identifying memory pressure or leaks in the Raft consensus layer, which can impact cluster stability and restore performance.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 461, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Errors", "group": "Backup & Import", "used_for": "Used for tracking backup operation errors in TiKV instances, broken down by error type. Essential for diagnosing backup failures and identifying which TiKV nodes are experiencing issues during backup processes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, error) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 459, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU utilization of TiKV backup worker threads and I/O operations during backup processes. Essential for identifying backup performance bottlenecks, detecting when backup operations consume excessive CPU resources, and verifying that CPU usage stays within configured soft limits.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 445, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cache Events", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of cache events during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose cache behavior and performance during data import and restore processes, particularly useful when troubleshooting slow restore operations or cache-related bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 50, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID QPS", "group": "Meta", "used_for": "Used for monitoring the rate of AutoID allocation operations per second in TiDB. Helps identify issues with auto-increment ID generation performance and diagnose bottlenecks when applications experience delays in obtaining new IDs for INSERT operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 180, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Client Unavailable Duration 95", "group": "Batch Client", "used_for": "Used for monitoring the 95th percentile duration when TiKV batch clients are unavailable in TiDB instances. Critical for diagnosing connection issues between TiDB and TiKV, helping identify network problems or TiKV unavailability that impacts query processing.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 181, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Task Status", "group": "GC", "used_for": "Used for monitoring the status and results of delete range tasks in TiDB's garbage collection process. Helps diagnose issues with space reclamation after table/index drops, showing task completion rates and potential failures that could lead to storage bloat.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*pipelined-dml.*\"}) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 453, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Speed", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of Point-in-Time Restore (PITR) apply requests being initiated on TiKV instances. Helps track restore operation progress and identify performance bottlenecks or stalled restore processes during data recovery scenarios.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 309, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing Internal Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of TiDB's internal transactions, such as those used for GC (garbage collection) and statistics updates. Critical for diagnosing when internal maintenance operations are taking too long and potentially impacting cluster performance or causing transaction conflicts.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"internal\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 460, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of backup threads allocated in each TiKV instance's backup thread pool. Essential for diagnosing backup performance issues and ensuring adequate thread resources are available during backup operations, especially when backup tasks are slow or timing out.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 451, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Batch Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of batch sizes during Point In Time Restore (PITR) apply operations in TiKV. Helps diagnose restore performance by showing how data is being written in batches, which is critical for identifying inefficient batch sizes that could slow down the recovery process.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "TiDB-Cluster-Clinic", "title": "GC Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) failures per minute in TiDB, categorized by failure type. Critical for identifying GC process issues that can lead to storage bloat, performance degradation, and potential cluster instability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 158, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Failure OPM", "group": "GC", "used_for": "Used for monitoring the per-minute failure rate of TiDB's unsafe destroy range operations during garbage collection. Critical for identifying issues with physical data deletion in TiKV, which can lead to disk space not being reclaimed properly or GC process failures.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 162, "dashboard": "TiDB-Cluster-Clinic", "title": "New ETCD Session Duration 95", "group": "Owner", "used_for": "Used for monitoring the 95th percentile latency of establishing new ETCD sessions in TiDB's owner election mechanism. Critical for diagnosing coordination issues and detecting ETCD connectivity problems that could affect DDL operations and cluster metadata management.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 435, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fail advance ts count", "group": "Resolved TS", "used_for": "Used for monitoring failures in advancing the resolved timestamp in TiKV, which is critical for maintaining data consistency in distributed transactions. Helps diagnose issues with stale peer detection and resolved TS progression problems that can impact CDC (Change Data Capture) and transaction consistency across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572005, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Meta Updating Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile latency of statistics metadata updates in TiDB. Critical for diagnosing performance issues related to table statistics maintenance, which can impact query optimization and execution plan quality when updates are slow.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 454, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cached File in Memory", "group": "Point In Time Restore", "used_for": "Used for monitoring the amount of cached file data in memory during Point-in-Time Restore (PITR) operations on each TiKV instance. Helps identify memory pressure and cache efficiency when restoring data, particularly useful for diagnosing slow restore performance or out-of-memory issues during recovery operations.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 86, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99", "group": "GC", "used_for": "Used for monitoring the 99th percentile latency of TiDB's garbage collection (GC) operations. Essential for identifying GC performance degradation that could impact transaction processing and cluster stability, particularly when investigating slow queries or transaction timeouts.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 442, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "P99 RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the 99th percentile RPC latency during Point In Time Restore (PITR) operations in TiKV, including apply requests, download queue/execution, and engine operations. Critical for diagnosing performance bottlenecks and slow restore operations when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "TiDB-Cluster-Clinic", "title": "Meta Operations Duration 99", "group": "Meta", "used_for": "Used for monitoring the 99th percentile latency of TiDB metadata operations (such as schema changes, table creation, and DDL operations). Critical for diagnosing slow DDL execution and identifying metadata-related performance bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763571994, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID Client Conn Reset Counter", "group": "Meta", "used_for": "Used for monitoring the rate of AutoID client connection resets in TiDB's meta service. Helps diagnose issues with AutoID allocation stability and network connectivity problems between TiDB servers and the AutoID service, which can impact auto-increment ID generation.", "component": "TIDB", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_meta_autoid_client_conn_reset_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572006, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Loading Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile latency of statistics delta loading operations in TiDB. Critical for diagnosing performance issues related to table statistics updates, which can impact query optimization and execution plan generation when loading takes too long.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_load_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 90, "dashboard": "TiDB-Cluster-Clinic", "title": "Too Many Locks Error OPM", "group": "GC", "used_for": "Used for monitoring the rate of \"too many locks\" errors encountered during garbage collection operations in TiDB. This panel helps diagnose GC blocking issues caused by excessive uncommitted locks in regions, which can prevent old data cleanup and lead to storage bloat.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 163, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Watcher OPS", "group": "Owner", "used_for": "Used for monitoring the rate of owner watcher operations in TiDB's distributed coordination system. Helps diagnose issues with DDL owner election, schema synchronization, and cluster metadata management by tracking watcher operation throughput and success/failure rates across instances.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572009, "dashboard": "TiDB-Cluster-Clinic", "title": "Binding Cache Hit / Miss OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of SQL plan binding cache hits versus misses in TiDB. Helps evaluate the effectiveness of plan binding cache configuration and identify when bindings aren't being cached efficiently, which could impact query optimization performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_binding_cache_hit_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_binding_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 450, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Downloaded File Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of file sizes downloaded during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose restore performance issues by tracking download volume patterns and identifying unusually large file transfers that may impact recovery time.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 446, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Overall RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the latency distribution of Point-in-Time Restore (PITR) apply RPC operations across TiKV instances. Essential for diagnosing performance bottlenecks during backup restoration and identifying slow apply operations that may delay recovery time objectives.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 447, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read File into Memory Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the time taken to read and load backup files into memory during Point-in-Time Restore (PITR) operations in TiKV. Critical for diagnosing slow restore performance and identifying bottlenecks in the file download phase of backup recovery.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 228, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Receive Average Duration", "group": "Batch Client", "used_for": "Used for monitoring the average and P99 latency of batch receive operations between TiDB and TiKV clients. Critical for diagnosing network communication delays and identifying instances with slow batch RPC response times that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572008, "dashboard": "TiDB-Cluster-Clinic", "title": "Binding Cache Memory Usage", "group": "Statistics & Plan Management", "used_for": "Used for monitoring memory consumption of TiDB's SQL binding cache, which stores execution plan bindings. Essential for tracking cache memory usage against configured limits to prevent out-of-memory issues and optimize plan binding performance.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_mem_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_server_binding_cache_mem_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572010, "dashboard": "TiDB-Cluster-Clinic", "title": "Number of Bindings in Cache", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the number of SQL plan bindings currently stored in TiDB's cache. Helps track plan binding usage and diagnose issues when execution plans aren't being enforced as expected, or when cache capacity limits are reached.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_num_bindings{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 204, "dashboard": "TiDB-Cluster-Clinic", "title": "Wait Connection Establish Duration", "group": "Batch Client", "used_for": "Used for monitoring the 99.99th percentile latency when TiDB batch clients wait to establish connections to TiKV. Critical for diagnosing connection pool exhaustion, network issues, or TiKV overload that causes extreme delays in establishing new connections.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 430, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts in region leaders", "group": "Resolved TS", "used_for": "Used for monitoring the maximum time gap between the current time and resolved timestamp across region leaders in TiKV. Critical for diagnosing replication lag issues and identifying regions where transaction consistency guarantees may be delayed, particularly important for CDC (Change Data Capture) and follower read scenarios.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 432, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check leader duration", "group": "Resolved TS", "used_for": "Used for monitoring the latency of leader checking operations in TiKV's Resolved TS mechanism. Critical for diagnosing delays in timestamp resolution and identifying issues with Raft leader communication that could impact CDC (Change Data Capture) and backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 456, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time distribution of applying SST files during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose restore performance bottlenecks by tracking how long it takes to apply imported data to the storage engine.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 182, "dashboard": "TiDB-Cluster-Clinic", "title": "Push Task Duration 95", "group": "GC", "used_for": "Used for monitoring the 95th percentile latency of GC push tasks from TiDB to TiKV. Helps identify performance bottlenecks in garbage collection operations when GC tasks experience delays in being distributed to TiKV nodes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*pipelined-dml.*\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 431, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Leader Resolved TS Region", "group": "Resolved TS", "used_for": "Used for identifying which TiKV region has the minimum resolved timestamp among all leader regions on each instance. Critical for diagnosing resolved-ts lag issues and pinpointing specific regions that are blocking transaction consistency advancement in distributed transactions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 441, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring CPU usage of Point-in-Time Restore (PITR) SST file processing threads on TiKV instances. Essential for diagnosing performance bottlenecks during backup restoration operations and identifying instances with high CPU consumption during PITR recovery processes.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 433, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request region count", "group": "Resolved TS", "used_for": "Used for monitoring the 99th percentile number of regions included in CheckLeader requests in TiKV's Resolved TS mechanism. Helps identify potential performance issues when checking leader status across many regions, which can impact transaction consistency and CDC (Change Data Capture) performance.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 237, "dashboard": "TiDB-Cluster-Clinic", "title": "Historical Stats OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per minute (OPM) of historical statistics generation and dumping in TiDB, broken down by success/failure results. Essential for tracking the health of statistics collection processes and identifying issues with statistics maintenance that could impact query optimization.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[2m])) by (result)", "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 436, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock heap size", "group": "Resolved TS", "used_for": "Used for monitoring the memory consumption of the lock heap in TiKV's Resolved TS mechanism. Essential for diagnosing memory pressure issues related to transaction lock tracking and identifying instances with excessive lock accumulation that may impact Resolved TS advancement.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 89, "dashboard": "TiDB-Cluster-Clinic", "title": "Action Result OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) action results per minute in TiDB, categorized by operation type. Helps diagnose GC efficiency issues, identify failed GC operations, and track the health of TiDB's distributed garbage collection process that cleans up obsolete MVCC versions.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 51, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID Duration", "group": "Meta", "used_for": "Used for monitoring the latency of TiDB's auto-increment ID allocation operations at p80 and p99 percentiles. Critical for diagnosing performance issues when applications experience delays in obtaining auto-increment IDs, particularly during high-concurrency insert operations or when the auto-ID cache needs replenishment from PD.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 452, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blocked by Concurrency Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time spent waiting for concurrency permits during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose restore performance bottlenecks caused by concurrency control limitations when importing data.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 439, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending command size", "group": "Resolved TS", "used_for": "Used for monitoring the total size of pending commands in TiKV's Resolved TS channel. Helps identify memory pressure and potential bottlenecks when Resolved TS commands accumulate, which can impact CDC (Change Data Capture) and backup operations that rely on consistent snapshots.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 438, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observe region status", "group": "Resolved TS", "used_for": "Used for monitoring the distribution of region resolution statuses in TiKV's Resolved TS mechanism. Helps diagnose issues with CDC (Change Data Capture) and transaction consistency by tracking which regions are successfully resolving timestamps versus those experiencing delays or failures.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 455, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Engine Requests Unfinished", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of unfinished engine requests during Point In Time Restore (PITR) operations in TiKV. Helps identify bottlenecks or stuck requests in the import applier process, which is critical for diagnosing slow or failed restore operations.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 443, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of import RPC operations in TiKV during Point In Time Restore (PITR) operations. Essential for tracking restore progress and diagnosing performance bottlenecks when recovering data to a specific point in time, particularly focusing on apply operations and overall import throughput per instance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 87, "dashboard": "TiDB-Cluster-Clinic", "title": "Config", "group": "GC", "used_for": "Used for monitoring TiDB's garbage collection (GC) configuration parameters across the cluster. Helps verify GC settings are correctly applied and identify configuration inconsistencies that could affect data cleanup and storage management.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 434, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request size", "group": "Resolved TS", "used_for": "Used for monitoring the 99th percentile size of CheckLeader requests in TiKV's Resolved TS mechanism, measured in both bytes and item count. Essential for diagnosing abnormally large CheckLeader requests that could impact Resolved TS advancement performance and CDC/backup workload efficiency.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 444, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Count", "group": "Point In Time Restore", "used_for": "Used for monitoring the count of import RPC apply operations in TiKV during Point In Time Restore (PITR) processes. Helps track restore progress and diagnose issues when recovering data to a specific point in time by showing the volume of apply operations being processed across TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 437, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scan backoff duration", "group": "Resolved TS", "used_for": "Used for monitoring the backoff duration during initial scan operations in TiKV's Resolved TS mechanism. Helps diagnose delays in establishing resolved timestamps, which can indicate lock conflicts or resource contention affecting CDC (Change Data Capture) and backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 463, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in the default column family during TiKV backup operations. Helps diagnose backup performance issues and identify whether SST file sizes are within optimal ranges for efficient backup processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 297, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB CPU Usage", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiDB server instances, showing actual CPU usage rate against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks and resource saturation issues in TiDB nodes, particularly when investigating TTL (Time To Live) job performance impact.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 239, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Report Duration", "group": "TopSQL", "used_for": "Used for monitoring the 99th percentile latency of TopSQL report generation and submission in TiDB and TiKV. Critical for diagnosing performance issues with the TopSQL feature itself, helping identify when report collection overhead becomes excessive or when communication delays occur between components.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[5m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 464, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file generation during TiKV backup operations, measured in bytes per second. Essential for evaluating backup performance, identifying slow backup instances, and diagnosing bottlenecks in the backup process across different column families.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, cf) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 466, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time distribution of SST file scanning operations during TiKV backup processes. Essential for diagnosing backup performance issues and identifying slow scan operations that may impact backup completion time.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 299, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV CPU", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiKV instances in the context of TTL (Time To Live) operations. Helps identify if TTL background tasks are causing excessive CPU consumption on specific TiKV nodes, which could impact overall cluster performance.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\".*tikv\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 472, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO Utilization", "group": "Backup & Import", "used_for": "Used for monitoring disk I/O utilization percentage on TiKV nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup or bulk import processes, helping diagnose whether disk saturation is causing performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, device) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 471, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of checksum and analyze requests in TiKV's coprocessor, tracking both p99 and p9999 percentiles. Critical for diagnosing performance issues during backup operations, table checksums, and ANALYZE TABLE commands that verify data integrity.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 470, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of creating connections to external storage systems (S3, GCS, etc.) during backup and restore operations in TiKV. Critical for diagnosing slow backup/restore performance caused by external storage connection issues, with p99 and p9999 percentiles helping identify tail latency problems in storage initialization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 465, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time distribution of SST file scanning operations during TiKV backup processes. Essential for diagnosing backup performance issues and identifying when snapshot scanning is taking longer than expected, which could indicate I/O bottlenecks or large data volumes impacting backup efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 242, "dashboard": "TiDB-Cluster-Clinic", "title": "Report Data Count Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the volume of Top SQL data being reported by TiDB instances per minute, broken down by data type. Helps diagnose Top SQL feature health and identify potential issues with query performance data collection or reporting bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_report_data_total_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, job)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 305, "dashboard": "TiDB-Cluster-Clinic", "title": "Ignore Event Per Minute", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for tracking the rate of SQL statements ignored by TopSQL feature per minute, categorized by ignore reason type. Helps diagnose why certain queries are not being captured in TopSQL statistics, which is critical when troubleshooting missing query data or understanding TopSQL sampling behavior.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_ignored_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 467, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time taken to save SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup data persistence phase.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 301, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV IO MBps", "group": "TTL", "used_for": "Used for monitoring TiKV's I/O throughput in megabytes per second during TTL (Time To Live) operations, tracking WAL writes and data reads across instances. Essential for identifying I/O bottlenecks and uneven load distribution when TTL background tasks are deleting expired data, helping diagnose performance degradation caused by excessive disk I/O.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 243, "dashboard": "TiDB-Cluster-Clinic", "title": "Total Report Count", "group": "TopSQL", "used_for": "Used for monitoring the volume and success rate of TopSQL reports sent from TiDB and TiKV instances to the data sink. Essential for diagnosing TopSQL feature health, identifying report delivery failures, and ensuring query performance data is being collected properly for workload analysis.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)", "sum(increase(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, result)", "sum(tikv_resource_metering_report_data_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"sent\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 469, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of external storage connection creation in TiKV during backup and restore operations. Critical for diagnosing slow backup/restore performance caused by delays in establishing connections to external storage systems like S3 or GCS.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 241, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Profiling OPS", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CPU profiling operations in TiDB instances. Helps verify that CPU profiling is actively collecting performance data for TopSQL analysis and troubleshooting query performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_cpu_profile_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 473, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import operations. Essential for diagnosing performance bottlenecks and resource contention when importing large datasets or restoring backups, helping identify if import processes are CPU-bound or if thread pool sizing needs adjustment.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 474, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of active SST (Sorted String Table) import threads per TiKV instance during data import or restore operations. Essential for diagnosing import performance bottlenecks and verifying that backup/restore jobs are utilizing available thread resources efficiently.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL QPS By Type", "group": "TTL", "used_for": "Used for monitoring the queries per second (QPS) of TTL (Time To Live) operations in TiDB, broken down by SQL type and execution result. Helps identify TTL workload patterns, track the volume of expired data cleanup operations, and diagnose issues when TTL background jobs are running slower than expected or failing.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 287, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Processed Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows processed by TiDB's Time-To-Live (TTL) feature, broken down by SQL operation type and result status. Essential for tracking TTL job performance, identifying processing bottlenecks, and ensuring expired data is being cleaned up at the expected rate.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 475, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Errors", "group": "Backup & Import", "used_for": "Used for tracking import operation errors in TiKV during data backup and restore processes. Helps diagnose failures in bulk data import operations by showing error counts grouped by error type and instance, essential for troubleshooting data migration issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, error, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 477, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Backup & Import", "used_for": "Used for monitoring the rate of import RPC operations in TiKV, broken down by request type. Essential for tracking data import activity and diagnosing throughput issues during bulk data loading or restore operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 468, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, tracking p99 and p999 percentiles. Critical for diagnosing backup performance issues and identifying slow backup ranges that could impact backup completion times or cluster performance during backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 483, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets through tools like TiDB Lightning or BR restore operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 293, "dashboard": "TiDB-Cluster-Clinic", "title": "EMA CPU Usage", "group": "Resource Manager", "used_for": "Used for monitoring the Exponential Moving Average (EMA) of CPU usage in TiDB's Resource Manager. Helps identify CPU consumption trends and assess whether resource control policies are effectively managing workload resources over time.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 276, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) scan workers in TiDB. Helps identify which phase (scanning, deleting, or waiting) is consuming the most time, enabling diagnosis of TTL job performance bottlenecks and optimization opportunities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan_worker\"}[2m])) by (phase)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 281, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Job Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) jobs across different statuses in TiDB. Helps identify job execution patterns, detect stuck or failed TTL jobs, and troubleshoot data cleanup issues when expired data isn't being removed as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 300, "dashboard": "TiDB-Cluster-Clinic", "title": "Table Count By TTL Schedule Delay", "group": "TTL", "used_for": "Used for monitoring the scheduling delay of TTL (Time To Live) jobs across different tables in TiDB. Helps identify tables experiencing delays in TTL task scheduling, which can indicate resource contention or configuration issues affecting automatic data cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_server_ttl_watermark_delay{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", type=\"schedule\"}) by (type, name)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 303, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of expired rows successfully deleted by TTL (Time To Live) jobs per hour in TiDB. Helps track TTL cleanup efficiency and identify potential issues when expired data isn't being purged as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 479, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Essential for diagnosing performance bottlenecks when importing large datasets or restoring backups, helping identify whether write or download operations are causing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 462, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST files in the Write column family during TiKV backup operations. Helps diagnose backup performance issues and identify whether large SST files are causing backup latency or throughput bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 302, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows inserted per second by TiDB's Time-To-Live (TTL) feature. Helps identify TTL job activity levels and diagnose potential performance impacts when TTL operations are inserting data back into tables during cleanup processes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 313, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Event Count Per Minute", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) events occurring per minute in TiDB, broken down by event type. Essential for tracking TTL job activity, identifying abnormal deletion patterns, and diagnosing issues with automatic data expiration and cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"hook.tidb.ttl\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"runtime\\\\.ttl.*\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 289, "dashboard": "TiDB-Cluster-Clinic", "title": "GOGC", "group": "Resource Manager", "used_for": "Used for monitoring the GOGC (Go Garbage Collection) parameter value in TiDB instances. Helps diagnose memory management behavior and tune garbage collection frequency when investigating memory pressure or GC-related performance issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 482, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data or migrating large datasets, as rewrite operations can significantly impact import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 284, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Scan Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution (p50, p80, p90, p99) of TTL scan SELECT queries in TiDB. Essential for diagnosing performance issues with Time-To-Live background scanning operations and identifying when expired data cleanup is impacting query response times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))\n", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 298, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of rows inserted by TTL (Time To Live) operations per hour in TiDB. Helps track TTL job activity and identify unusual insertion patterns or performance issues with TTL-managed data lifecycle operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1h]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 487, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write keys", "group": "Backup & Import", "used_for": "Used for monitoring the number of keys written during TiDB Lightning's local import process, broken down by write type and TiKV instance. Helps diagnose import performance issues and verify data ingestion progress during bulk data loading operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 481, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Read SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of reading SST (Sorted String Table) files during data import operations in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the SST file download/read phase of backup restoration or data migration tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 484, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, as high ingestion latency directly impacts import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 294, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Task Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) tasks across different statuses in TiDB. Helps track task execution states (running, completed, failed) to identify TTL job bottlenecks or failures when automatic data cleanup is not working as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 480, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Wait Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time data import requests spend waiting in the queue before processing in TiKV. Critical for diagnosing import bottlenecks and identifying when the import subsystem is overloaded or experiencing queueing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 486, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during data import operations in TiKV. Helps diagnose import performance bottlenecks and network bandwidth issues when restoring backups or migrating data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 285, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution of TTL (Time To Live) delete queries in TiDB across multiple percentiles (p50, p80, p90, p99). Essential for diagnosing performance issues with automatic data expiration operations and identifying when TTL delete queries are experiencing slowdowns that could impact background cleanup efficiency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 493, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Status", "group": "Backup Log", "used_for": "Used for monitoring the status of backup log tasks in TiKV instances. Helps quickly identify which instances have backup log tasks in abnormal states, enabling rapid detection of backup failures or interruptions that could impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 310, "dashboard": "TiDB-Cluster-Clinic", "title": "Closest Replica Hit Count Per Second", "group": "Follower Read", "used_for": "Used for monitoring the rate of queries that successfully hit the closest replica when follower read is enabled in TiDB. Helps verify follower read effectiveness and diagnose whether queries are properly utilizing nearby replicas to reduce cross-AZ network traffic and improve read performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_closest_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 490, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "cloud request", "group": "Backup & Import", "used_for": "Used for monitoring the rate of cloud storage requests (such as S3 or GCS operations) made by TiKV instances during backup and import operations. Helps identify cloud API call patterns and diagnose issues with external storage connectivity or throughput during data backup/restore processes.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cloud, req) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 488, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write bytes", "group": "Backup & Import", "used_for": "Used for monitoring the write throughput (bytes per second) during TiDB Lightning's local import operations to TiKV instances. Helps diagnose import performance bottlenecks and verify data ingestion rates when using the local backend for bulk data loading.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 492, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Endpoint Status", "group": "Backup Log", "used_for": "Used for monitoring the operational status of TiKV's log backup endpoint, indicating whether log backup is currently enabled or disabled on each instance. Essential for verifying backup configuration and quickly identifying instances where log backup may have been inadvertently disabled or failed to start.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 478, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Count", "group": "Backup & Import", "used_for": "Used for monitoring the count of import RPC requests in TiKV's backup and import operations. Helps track import activity volume and identify potential bottlenecks or failures during data import processes, especially when diagnosing slow or stalled import jobs.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) delete worker operations in TiDB. Helps identify performance bottlenecks in the TTL deletion process by showing which phase (e.g., scanning, deleting, waiting) consumes the most time.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"delete_worker\"}[2m])) by (phase)\n" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 489, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL Expired", "group": "Backup & Import", "used_for": "Used for monitoring the count of expired Time-To-Live (TTL) entries in TiKV's backup raw data. Helps diagnose backup data retention issues and identify instances where stale backup data is being cleaned up, which can impact backup storage management and performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 476, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of import RPC operations in TiKV during data import processes, tracking both p99 and p9999 percentiles. Critical for diagnosing performance bottlenecks during bulk data imports or restore operations, helping identify slow import requests that could delay data migration tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 485, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "Backup & Import", "used_for": "Used for monitoring the distribution of SST (Sorted String Table) file sizes being ingested during data import operations in TiKV. Helps diagnose import performance issues by revealing whether large SST files are causing ingestion bottlenecks or delays during backup restoration and bulk data loading scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 324, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size Per Instance and TiKV", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor response data size from each TiKV store to TiDB instances during follower read operations. Helps identify network bandwidth usage patterns and diagnose potential data transfer bottlenecks or imbalanced read traffic distribution across TiKV stores.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 245, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV Stat Task OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of resource metering statistics tasks processed by TiKV instances. Helps diagnose TopSQL feature performance and identify issues with resource consumption tracking when analyzing query-level resource usage patterns.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_resource_metering_stat_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 325, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size", "group": "Follower Read", "used_for": "Used for monitoring the size of coprocessor responses returned from TiKV to TiDB, showing average, median (p50), p80, and p99 percentiles. Essential for diagnosing network bandwidth issues, identifying queries that transfer excessive data, and understanding the I/O characteristics of follower read operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_distsql_copr_resp_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.5, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 494, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advancer Owner", "group": "Backup Log", "used_for": "Used for identifying which TiKV node is currently serving as the log backup advancer owner. Essential for troubleshooting log backup coordination issues and verifying proper leadership election in the backup subsystem.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "tidb_log_backup_advancer_owner > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "TiDB-Cluster-Clinic", "title": "Delivered kv count", "group": "Import Into", "used_for": "Used for monitoring the total number of key-value pairs delivered during IMPORT INTO operations in TiDB. Helps track data ingestion progress and diagnose slow import performance by identifying bottlenecks in the KV delivery pipeline.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_block_deliver_kv_pairs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 495, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Flush Size", "group": "Backup Log", "used_for": "Used for monitoring the average size of log backup files flushed to storage by each TiKV instance. Helps identify backup efficiency issues and detect instances producing abnormally large or small flush sizes, which may indicate configuration problems or uneven workload distribution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 496, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flushed Files (Last 30m) Per Host", "group": "Backup Log", "used_for": "Used for monitoring the number of log backup files flushed by each TiKV instance over the last 30 minutes. Helps verify backup log activity and identify instances with abnormal flush patterns or potential backup pipeline issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 319, "dashboard": "TiDB-Cluster-Clinic", "title": "Data read/encode/deliver average duration", "group": "Import Into", "used_for": "Used for monitoring the average latency of three key stages in TiDB's IMPORT INTO operation: reading source data, encoding rows, and delivering data blocks. Essential for diagnosing performance bottlenecks during bulk data import operations and identifying which stage is causing slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_import_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_row_read_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_read_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 499, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Files (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the number of log backup files flushed to external storage by each TiKV instance over the last 30 minutes. Helps diagnose backup log performance issues, identify instances with abnormal flush rates, and verify that log backup is progressing normally across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 317, "dashboard": "TiDB-Cluster-Clinic", "title": "Encoded/Delivered/Imported data size", "group": "Import Into", "used_for": "Used for monitoring the data flow during IMPORT INTO operations in TiDB, tracking three key stages: encoded data size (total_restore), data written to storage (written), and data successfully restored (restored). Essential for diagnosing import performance bottlenecks, verifying data processing progress, and identifying discrepancies between encoded, written, and imported data volumes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"total_restore\"}) by(task_id)", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 497, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Times (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the number of log backup flush operations completed in the last 30 minutes per TiKV instance. Helps identify backup activity patterns and diagnose issues when backup flush operations are missing or abnormally frequent during the file-saving stage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 498, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Flushed Size (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total amount of backup log data flushed to storage by each TiKV instance over the last 30 minutes. Helps assess backup log throughput, identify instances with abnormal flush volumes, and diagnose backup performance issues or storage bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 501, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle Event Rate", "group": "Backup Log", "used_for": "Used for monitoring the rate at which TiKV processes backup log events per instance. Essential for tracking backup log throughput and identifying instances with slow or stalled log backup processing that could lead to backup delays.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571996, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Duration", "group": "Global Sort", "used_for": "**Used For:**\n\nUsed for monitoring the latency of writing data to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage, particularly when investigating slow data import or DDL operations using global sort.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 316, "dashboard": "TiDB-Cluster-Clinic", "title": "Total encode/deliver/import-kv speed", "group": "Import Into", "used_for": "Used for monitoring the throughput of IMPORT INTO operations across three stages: data encoding (restored), delivery to TiKV (written), and final import completion (imported). Essential for diagnosing performance bottlenecks during bulk data import tasks and identifying which stage is limiting overall import speed.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}[2m])) by(task_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 320, "dashboard": "TiDB-Cluster-Clinic", "title": "Import/Normal mode", "group": "Import Into", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration during IMPORT INTO operations. Helps verify if TiKV switches between import mode (higher limit for faster bulk loading) and normal mode (standard limit for regular operations), which is critical for diagnosing import performance issues and mode transition problems.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572000, "dashboard": "TiDB-Cluster-Clinic", "title": "Ingest Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active ingest workers handling global sort operations in TiDB. Helps diagnose performance bottlenecks and resource utilization issues during large-scale data import or DDL operations that require global sorting.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_ingest_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 502, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan Generate Event Throughput", "group": "Backup Log", "used_for": "Used for monitoring the throughput of initial scan event generation during TiKV log backup operations. Helps diagnose backup performance issues by tracking how quickly TiKV instances scan and generate backup events during the initial backup phase.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 304, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert/Delete Rows By Day", "group": "TTL", "used_for": "Used for monitoring the daily volume of rows inserted and deleted by TiDB's Time-To-Live (TTL) feature. Essential for tracking TTL job effectiveness, verifying expired data cleanup rates, and identifying potential issues with automatic data lifecycle management that could impact storage growth or performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 504, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Of Events", "group": "Backup Log", "used_for": "Used for monitoring the heap memory consumption of backup log events in TiKV instances. Essential for tracking memory usage during log backup operations and preventing out-of-memory issues when backup workloads are active.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 510, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scanning duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to complete initial log scanning during TiKV backup operations. Critical for diagnosing slow backup startup issues and identifying performance bottlenecks in the backup log initialization phase.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 500, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Backup Log", "used_for": "Used for monitoring CPU usage of TiKV's backup log threads, including backup stream and log backup scan workers. Essential for identifying resource bottlenecks during continuous backup operations and diagnosing performance issues in log backup processes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 507, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fatal Errors", "group": "Backup Log", "used_for": "Used for monitoring fatal errors occurring in TiKV's log backup feature, broken down by error type and instance. Critical for identifying backup failures and data protection issues that require immediate attention to prevent data loss.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 336, "dashboard": "TiDB-Cluster-Clinic", "title": "Parallel upload worker count", "group": "Global Sort", "used_for": "Used for monitoring the number of active parallel workers uploading sorted data during TiDB's global sort operations. Helps diagnose global sort performance issues by identifying whether upload parallelism is optimal or if worker count bottlenecks are causing latency problems.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_upload_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 509, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to flush backup log files to external storage in TiKV. Critical for diagnosing backup performance issues and identifying bottlenecks in the log backup pipeline's file-saving stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and whether servers have been up long enough to provide meaningful performance metrics.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 23763571997, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the write throughput rate (at p50, p95, and p999 percentiles) when TiDB's Global Sort feature writes data to cloud storage during distributed operations like index creation or data import. Critical for diagnosing performance bottlenecks in cloud storage write operations and identifying whether cloud storage bandwidth or latency is impacting large-scale data processing tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 5, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring CPU utilization of TiDB server instances compared to their configured CPU limits (maxprocs). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiDB instances need vertical scaling or if query workloads need optimization.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 4, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances. Essential for tracking connection usage patterns, identifying connection leaks, and diagnosing issues related to connection pool exhaustion or unexpected connection spikes that may impact database availability.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 506, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errors", "group": "Backup Log", "used_for": "Used for monitoring the rate of errors occurring in TiKV's log backup process, broken down by error type and instance. Essential for quickly identifying backup failures, diagnosing log backup reliability issues, and determining which TiKV nodes are experiencing backup problems.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 511, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Convert Raft Event duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent converting Raft events to stream events during TiKV log backup operations. Critical for diagnosing backup performance bottlenecks and identifying delays in the event conversion pipeline that could impact backup throughput and RPO objectives.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571998, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of reading data from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, particularly when investigating slow data import or index building operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763571999, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the read throughput rate from cloud storage during TiDB's global sort operations at different percentiles (p50, p95, p999). Essential for diagnosing performance bottlenecks in cloud-based data import and sort operations, helping identify slow cloud storage read patterns that may impact large-scale data processing tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 517, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Type", "group": "Backup Log", "used_for": "Used for monitoring the distribution of internal message types processed by TiKV's log backup actor. Helps diagnose log backup performance issues by identifying which message types are most frequently processed and potential bottlenecks in the backup pipeline.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 512, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait for Lock Duration", "group": "Backup Log", "used_for": "Used for monitoring the time TiKV backup log operations spend waiting to acquire router locks. Critical for diagnosing backup performance bottlenecks caused by lock contention, especially when backup tasks are slow or experiencing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 503, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Abnormal Checkpoint TS Lag", "group": "Backup Log", "used_for": "Used for monitoring the time lag between the current time and the last backup checkpoint timestamp in TiDB's log backup feature. Critical for detecting backup delays or stalled backup progress that could impact point-in-time recovery capabilities and data protection SLAs.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Failed Query OPS", "group": "", "used_for": "Used for monitoring the rate of failed queries in TiDB, broken down by error type and instance. Essential for quickly identifying query execution issues, detecting error spikes, and diagnosing which types of failures (e.g., syntax errors, permission issues, or execution timeouts) are affecting the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 516, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "System Write Call Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of system write calls during TiKV log backup operations. Essential for diagnosing I/O performance bottlenecks and identifying when backup processes are experiencing slow disk writes that could impact cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 515, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent writing backup log data to temporary files in TiKV. Critical for diagnosing backup performance bottlenecks and identifying slow disk I/O issues during the log backup process.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 518, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile latency of internal message handling in TiKV's backup log actor system. Critical for diagnosing performance bottlenecks in log backup operations and identifying when internal message processing becomes slow or blocked.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 505, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observed Region Count", "group": "Backup Log", "used_for": "Used for monitoring the number of regions being observed by the backup log feature on each TiKV instance. Helps diagnose backup coverage issues and identify instances that may be missing region data during log backup operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiDB server memory consumption, including process resident memory, Go heap usage, and configured memory quota. Essential for detecting memory leaks, OOM risks, and ensuring TiDB instances operate within their allocated memory limits.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 513, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Command Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of batch sizes when TiKV processes key-value pairs during log backup operations. Helps identify batching efficiency and diagnose backup performance issues by showing whether commands are being processed in optimal batch sizes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Duration", "group": "", "used_for": "Used for monitoring TiDB query execution latency at P95 and P99 percentiles, along with average duration. Essential for identifying query performance degradation, detecting slow queries, and diagnosing whether latency issues affect all queries or just tail latencies during performance troubleshooting.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 11, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS By Instance", "group": "", "used_for": "Used for monitoring the Commands Per Second (CPS) executed by each TiDB instance in the cluster. Helps identify load distribution imbalances across TiDB servers and pinpoint instances experiencing abnormal traffic patterns or performance bottlenecks.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 519, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile latency of internal message processing in TiKV's backup log actor system. Critical for diagnosing performance bottlenecks in log backup operations and identifying when backup processes are experiencing delays in handling internal coordination messages.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "QPS", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for tracking cluster workload patterns, identifying traffic spikes, and understanding the distribution of different query types during performance analysis or capacity planning.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 521, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Operation", "group": "Backup Log", "used_for": "Used for monitoring the rate of RocksDB operations during TiKV log backup's initial scan phase, broken down by column family and operation type. Essential for diagnosing performance bottlenecks and tracking progress during the initial backup scan process.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 514, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Save to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent saving backup log data to temporary files in TiKV's log backup process. Critical for diagnosing backup performance bottlenecks and identifying I/O-related issues during the log backup operation's file writing stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Parse Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL parsing latency in TiDB. Helps identify performance bottlenecks in the query parsing phase, particularly when dealing with complex SQL statements or syntax issues that cause excessive parsing overhead.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 17, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Execution Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of general SQL queries in TiDB. Essential for identifying query performance degradation and tail latency issues that affect user experience, helping diagnose slow query problems before they impact the majority of requests.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 13, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "", "used_for": "Used for monitoring the rate of commands per second (CPS) executed in TiDB, broken down by command type (SELECT, INSERT, UPDATE, DELETE, etc.). Helps identify workload composition, detect unusual command patterns, and understand which types of operations are dominating cluster activity.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 20, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "TPS", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring transactions per second (TPS) in TiDB, broken down by transaction type and mode (optimistic/pessimistic). Essential for tracking overall transaction throughput, identifying workload patterns, and detecting sudden changes in transaction volume that may indicate application issues or load shifts.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 524, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Key Putting", "group": "Backup Log", "used_for": "Used for monitoring the rate of region checkpoint metadata key operations during TiKV log backup. Helps diagnose backup performance issues by tracking how frequently checkpoint keys are being written to track backup progress across regions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 520, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Throughput", "group": "Backup Log", "used_for": "Used for monitoring the read throughput during TiKV's initial scan phase of log backup operations, broken down by RocksDB column family. Essential for diagnosing performance bottlenecks during backup initialization and understanding how efficiently data is being read from storage.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 522, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Trigger Reason", "group": "Backup Log", "used_for": "Used for tracking the reasons that trigger initial scanning operations in TiKV's backup log feature. Helps diagnose why backup scans are initiated (e.g., region changes, leader transfers, or configuration updates) and identify potential issues causing excessive or unexpected scan triggers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 523, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Task Status", "group": "Backup Log", "used_for": "Used for monitoring the status and progress of initial scanning tasks during TiKV log backup operations. Helps identify bottlenecks in the backup initialization phase by tracking pending scan tasks across different stages, essential for diagnosing slow backup startup or stalled backup processes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (stage) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Command Per Second", "group": "", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (OK vs errors). Essential for tracking overall database throughput, identifying error spikes, and comparing current query rates against historical baselines to detect traffic anomalies or performance degradation.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 21, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Transaction Duration", "group": "", "used_for": "Used for monitoring transaction execution latency at different percentiles (p80, p95, p99) across TiDB instances. Essential for identifying transaction performance degradation, detecting slow transaction patterns, and diagnosing issues affecting user-facing transaction response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 18, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Query Detail", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache, broken down by cache type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 525, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Checkpoint Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of batch sizes when TiKV processes checkpoint requests during log backup operations. Helps identify batching efficiency and diagnose potential bottlenecks in the backup log checkpoint advancement process.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 16, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Compile Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL compilation latency in TiDB, measuring the time spent parsing and optimizing queries before execution. Essential for diagnosing slow query performance caused by complex SQL statements or inefficient query plan generation.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 23, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Retry Num", "group": "", "used_for": "Used for monitoring the maximum number of retries for transactions in TiDB due to conflicts or lock timeouts. Critical for identifying transaction contention issues and optimistic locking problems that may indicate hot spots or inefficient transaction patterns requiring query optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 526, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of backup log advancement ticks in TiKV. Helps diagnose performance issues with the log backup process, particularly when backup operations are slow or experiencing delays in advancing the backup checkpoint.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Statement Num", "group": "", "used_for": "Used for monitoring the maximum number of SQL statements executed within a single transaction across TiDB instances. Helps identify transactions with excessive statement counts that may cause performance issues, long-running transactions, or potential application design problems requiring transaction splitting.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 527, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Failure Reason", "group": "Backup Log", "used_for": "Used for tracking the failure rate of region checkpoint operations during TiDB log backup, categorized by failure reason. Essential for diagnosing backup reliability issues and identifying specific error patterns that prevent regions from being successfully checkpointed.", "component": "TIKV", "phenomenon": "error_rate", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 528, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Result", "group": "Backup Log", "used_for": "Used for monitoring the rate and distribution of log backup region requests by their result status (success/failure) in TiKV. Helps identify backup operation health and diagnose issues when backup requests are failing or experiencing errors.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 532, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Try Advance Trigger Time", "group": "Backup Log", "used_for": "Used for monitoring the frequency of backup log advancement attempts in TiKV's log backup feature. Helps diagnose backup progress issues and identify when the backup advancer is actively trying to move the checkpoint forward.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 529, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile duration of backup log advancement ticks in TiKV. Critical for diagnosing performance issues with log backup operations and identifying when the backup advancer process experiences delays that could impact recovery point objectives (RPO).", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 531, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get Region Operation Count", "group": "Backup Log", "used_for": "Used for monitoring the frequency of region retrieval operations during TiKV backup log processing. Helps diagnose backup performance issues by tracking how often the backup advancer needs to fetch region information, which can indicate region distribution problems or backup bottlenecks.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "Dataplane Resource Overview", "title": "Pod Restart Count", "group": "", "used_for": "Used for tracking the cumulative restart count of pods in the dataplane namespace. Essential for identifying unstable pods experiencing crashes or OOM kills, which can indicate resource constraints, configuration issues, or application bugs affecting cluster stability.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "cluster", "control_plane_info", "namespace" ], "promql": [ "sum(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum(kube_pod_container_status_restarts_total{control_plane_info=\"$control_plane_info\",cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 38, "dashboard": "Dataplane Resource Overview", "title": "Pod Last Terminated Reason", "group": "", "used_for": "Used for tracking why pods in the dataplane namespace were last terminated, excluding normal completions. Essential for diagnosing pod crashes, OOM kills, and other abnormal termination events that may indicate resource constraints or application failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", reason!=\"Completed\"}) by (pod,reason)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 42, "dashboard": "Dataplane Resource Overview", "title": "Pod Abnormal Phase", "group": "", "used_for": "Used for identifying pods in abnormal states (not Running or Succeeded) within the specified namespace. Critical for quickly detecting pod failures, crashes, or scheduling issues that require immediate attention in the dataplane cluster.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\",namespace=\"$namespace\",phase!~\"Running|Succeeded\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 26, "dashboard": "Dataplane Resource Overview", "title": "Node Load 1m", "group": "Node", "used_for": "Used for monitoring the 1-minute load average per CPU core across cluster nodes. Helps identify nodes experiencing high system load or resource contention that could impact TiDB component performance and stability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_load1_per_cpu:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 36, "dashboard": "Dataplane Resource Overview", "title": "CPU Throttle", "group": "", "used_for": "Used for monitoring CPU throttling rates of containers in the dataplane namespace. Helps identify when pods are hitting CPU limits and experiencing performance degradation due to resource constraints, indicating potential need for CPU quota adjustments.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 530, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile duration of backup log advancement ticks in TiKV. Essential for diagnosing performance issues with log backup operations and identifying when the backup advancer process is experiencing delays that could impact recovery point objectives (RPO).", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Usage", "group": "Node", "used_for": "Used for monitoring CPU utilization across individual nodes in the dataplane cluster. Helps identify nodes experiencing high CPU load, enabling quick detection of resource bottlenecks and uneven workload distribution across the infrastructure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_cpu_utilisation:rate1m{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 30, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Usage", "group": "Node", "used_for": "Used for monitoring memory utilization ratio across individual nodes in the dataplane cluster. Helps identify nodes experiencing memory pressure or imbalanced memory distribution that could impact workload performance and stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_memory_utilisation:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 2, "dashboard": "Dataplane Resource Overview", "title": "Nodes", "group": "Node", "used_for": "Used for monitoring the distribution of Kubernetes nodes by instance type in the cluster. Helps operators understand cluster composition and identify potential resource allocation or scaling issues across different node types.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_beta_kubernetes_io_instance_type)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 540, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "group": "Memory", "used_for": "Used for monitoring the rate of memory allocation and deallocation across TiKV threads, showing allocated bytes as positive values and released bytes as negative values. Essential for identifying memory leak patterns, understanding per-thread memory pressure, and diagnosing out-of-memory issues in TiKV instances.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 539, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Allocator Stats", "group": "Memory", "used_for": "Used for monitoring memory allocation statistics in TiKV instances, broken down by allocation type. Essential for diagnosing memory usage patterns, identifying memory leaks, and understanding which components are consuming memory resources.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested memory resources to allocatable memory capacity on each Kubernetes node. Helps identify nodes approaching memory resource limits and potential scheduling constraints before pods fail to deploy due to insufficient memory requests.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",unit=\"byte\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 535, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes per second) across TiKV threads, showing the top 20 threads with I/O activity exceeding 1KB/s. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or abnormal disk usage patterns in TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 536, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring voluntary context switches across TiKV threads, which occur when threads willingly yield the CPU (typically during I/O waits). High rates indicate threads are frequently blocking on I/O operations, helping diagnose I/O bottlenecks and thread scheduling inefficiencies in TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 71, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested CPU resources to allocatable CPU capacity on each Kubernetes node. Helps identify nodes approaching resource limits and potential scheduling constraints that could prevent new pod deployments.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 544, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Arena Count", "group": "Memory", "used_for": "Used for monitoring the number of memory arenas allocated by TiKV's memory allocator (jemalloc) on each instance. Helps diagnose memory fragmentation issues and abnormal memory allocation patterns that could indicate memory leaks or inefficient memory usage.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4, "dashboard": "Dataplane Resource Overview", "title": "StatefulSet", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for StatefulSets in the selected namespace. Helps verify StatefulSet scaling configurations and detect discrepancies between desired and actual replica counts during deployment or scaling operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 547, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Priority Quota Limit", "group": "Resource Control", "used_for": "Used for monitoring the I/O quota limits assigned to different priority levels in TiKV's resource control system. Essential for verifying resource allocation configurations and diagnosing whether priority-based throttling is properly enforced across instances.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, priority) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 534, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the distribution of thread states (running, sleeping, waiting, etc.) across TiKV instances. Helps diagnose thread contention, resource starvation, and abnormal thread behavior that may indicate I/O bottlenecks or deadlock situations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "Dataplane Resource Overview", "title": "Root Disk Usage", "group": "Node", "used_for": "**Used For:**\n\nUsed for monitoring root filesystem disk space utilization across cluster nodes. Essential for identifying nodes approaching disk capacity limits, which can cause service disruptions, write failures, or system instability if storage becomes full.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "1 - sum(node_filesystem_avail_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device) / sum(node_filesystem_size_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 6, "dashboard": "Dataplane Resource Overview", "title": "Deployment", "group": "Pod", "used_for": "Used for monitoring the desired replica count configured for deployments in the specified Kubernetes namespace. Helps verify deployment scaling configurations and identify discrepancies between intended and actual pod counts during capacity planning or troubleshooting deployment issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_deployment_spec_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 537, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches across TiKV threads, which occur when the OS forcibly preempts threads due to CPU contention or time slice expiration. High rates (>100/sec) indicate CPU saturation or resource contention, helping diagnose performance degradation caused by insufficient CPU resources or thread scheduling issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 546, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Background Task Total Wait Duration", "group": "Resource Control", "used_for": "Used for monitoring the total time background tasks spend waiting due to resource control throttling in TiKV. Critical for diagnosing resource contention issues and identifying when background operations (like compaction or GC) are being delayed by resource quotas, which can impact overall cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, resource_group) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 542, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Released Bytes Rate per Thread", "group": "Memory", "used_for": "Used for monitoring the rate at which memory is being released (deallocated) by different threads in TiKV. Helps identify memory management patterns and diagnose potential memory leaks or inefficient deallocation in specific thread pools.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 543, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Mapped Allocation per Thread", "group": "Memory", "used_for": "Used for monitoring memory mapped allocations across different TiKV threads. Helps identify which threads are consuming the most mapped memory and diagnose memory pressure or allocation imbalances at the thread level.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Usage", "group": "Pod", "used_for": "Used for monitoring the working set memory usage of individual pods in the TiDB cluster. Helps identify memory-intensive pods, detect memory leaks, and diagnose out-of-memory (OOM) issues before they impact cluster stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 20, "dashboard": "Dataplane Resource Overview", "title": "DaemonSet", "group": "Pod", "used_for": "Used for monitoring the number of pods currently scheduled for each DaemonSet in the selected namespace. Helps verify that DaemonSets are properly deployed across nodes and identify scheduling issues when the actual pod count doesn't match expected node coverage.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_daemonset_status_current_number_scheduled{cluster=\"$cluster\",namespace=\"$namespace\"}) by (daemonset)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 46, "dashboard": "Dataplane Resource Overview", "title": "Pod Status", "group": "Pod", "used_for": "Used for monitoring the current status phase of pods in the specified namespace. Helps quickly identify pod health issues by showing which pods are in Running, Pending, Failed, or other phases, enabling rapid detection of deployment or scheduling problems.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 541, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Allocated Bytes Rate per Thread", "group": "Memory", "used_for": "Used for monitoring the memory allocation rate per thread in TiKV instances. Helps identify which threads are consuming memory most rapidly, useful for diagnosing memory pressure issues and detecting potential memory leaks in specific TiKV components.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 50, "dashboard": "Dataplane Resource Overview", "title": "Memory Utilization", "group": "Pod", "used_for": "Used for monitoring memory utilization percentage of pods in the dataplane by comparing actual memory usage (working set) against configured resource limits. Helps identify pods approaching memory limits, potential OOM risks, and opportunities for resource allocation optimization.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\"})by(pod)/\nsum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}\n)\n/\nsum by (pod) (\n kube_pod_container_resource_limits{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"POD\"}\n)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 508, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checkpoint TS of Tasks", "group": "Backup Log", "used_for": "**Used For:**\n\nUsed for monitoring the checkpoint timestamp of TiDB log backup tasks in TiKV, showing how current the backup data is relative to real-time. Essential for verifying backup freshness and identifying backup lag issues that could impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Request", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU resource requests and limits configured for each pod in the dataplane namespace. Essential for capacity planning, identifying pods with insufficient resource allocations, and detecting potential CPU throttling issues before they impact performance.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 550, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Status API Request (op/s)", "group": "Status Server", "used_for": "Used for monitoring the rate of HTTP requests to TiKV's status server API endpoints. Helps identify which diagnostic or monitoring endpoints are being accessed most frequently and detect unusual API usage patterns that could indicate monitoring issues or excessive polling.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Current Usage", "group": "Volume", "used_for": "Used for monitoring the current storage consumption of each PersistentVolume in the dataplane namespace. Helps identify volumes approaching capacity limits and diagnose storage-related issues before they impact cluster operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim) - sum(kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 10, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Requests", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring memory resource requests and limits configured for pods in the dataplane namespace. Helps identify pod memory allocation settings and detect potential resource constraint issues when actual usage approaches or exceeds configured requests/limits.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 48, "dashboard": "Dataplane Resource Overview", "title": "CPU Utilization", "group": "Pod", "used_for": "Used for monitoring CPU utilization of pods in the dataplane, showing both actual usage against limits and resource requests against node allocatable capacity. Essential for identifying CPU bottlenecks, over-provisioned pods, and node resource pressure in the cluster.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)/\nsum(kube_pod_container_resource_limit_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\"}[$__rate_interval]\n ) and on (container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)\n/\nsum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)", "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 14, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Usage", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of individual pods within the selected namespace. Helps identify pods with high CPU consumption, detect resource bottlenecks, and troubleshoot performance issues at the pod level.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (cluster,namespace,pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\", container!=\"POD\",container!=\"\"}[$__rate_interval])) " ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the disk space consumed by TiKV's encryption metadata files. Helps identify storage overhead from encryption-at-rest and detect potential issues with metadata file growth that could impact disk usage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in each TiKV instance. Helps track encryption key management overhead and identify potential issues with key storage or rotation in encrypted TiKV clusters.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 549, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Status API Request Duration", "group": "Status Server", "used_for": "Used for monitoring the latency of TiKV status API requests at p99 and p9999 percentiles. Essential for diagnosing slow responses from TiKV's HTTP status server, which provides metrics, configuration, and diagnostic endpoints used by monitoring systems and administrative tools.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Size", "group": "Volume", "used_for": "Used for monitoring the requested storage capacity of PersistentVolumeClaims in the specified namespace. Helps track storage resource allocation and identify potential capacity planning needs or storage quota issues in the Kubernetes cluster.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_resource_requests_storage_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 553, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the total number of encrypted files in each TiKV instance's storage. Helps verify encryption-at-rest is properly enabled and track the volume of encrypted data, useful when diagnosing encryption configuration issues or auditing data security compliance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 554, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether encryption-at-rest has been successfully initialized on TiKV instances. Essential for confirming that data encryption is properly enabled and operational, particularly after cluster setup or encryption configuration changes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 556, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypt/decrypt data nanos", "group": "Encryption", "used_for": "Used for monitoring the time spent on encryption and decryption operations in TiKV's RocksDB storage layer. Critical for diagnosing performance overhead when encryption-at-rest is enabled and identifying whether encryption operations are becoming a bottleneck for I/O operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 53, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message processing in TiKV instances. Critical for identifying slow gRPC operations and diagnosing communication bottlenecks between TiDB/TiKV components, excluding garbage collection operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99, tikv_grpc_msg_duration_seconds_bucket:le_type:sum:rate{type!=\"kv_gc\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV, broken down by message type. Critical for identifying communication failures between TiDB/TiKV components and diagnosing cluster stability issues or network problems.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 557, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the latency of reading and writing encryption metadata files in TiKV's encryption-at-rest feature. Critical for diagnosing performance issues related to encryption key management operations, especially when experiencing slow I/O or encryption-related bottlenecks affecting overall TiKV performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC message processing in TiKV, broken down by message type and priority. Essential for diagnosing communication bottlenecks between TiDB/TiKV components and identifying which specific gRPC operations are experiencing performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "Dataplane Resource Overview", "title": "Docker Images", "group": "Image Versions", "used_for": "Used for monitoring the container images deployed across pods in the dataplane namespace. Helps track image versions, identify outdated containers, and verify deployment consistency across the cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_pod_container_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 56, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages sent between TiKV nodes over gRPC. Helps diagnose Raft communication efficiency and identify potential network batching issues that could impact cluster consensus performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 67, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb)", "group": "", "used_for": "Used for monitoring Horizontal Pod Autoscaler (HPA) status for TiDB instances, showing desired, current, and maximum replica counts. Essential for tracking auto-scaling behavior and ensuring TiDB pods scale appropriately based on workload demands.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 55, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile and average batch sizes of gRPC requests and responses in TiKV. Essential for diagnosing gRPC batching efficiency and identifying potential network communication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 561, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL check progress", "group": "TTL", "used_for": "Used for monitoring the TTL (Time To Live) checker's progress rate across TiKV instances by tracking the ratio of processed regions to total regions. Helps identify TTL scanning bottlenecks and ensures expired data is being cleaned up efficiently across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolumeClaim", "group": "Volume", "used_for": "Used for displaying information about PersistentVolumeClaims (PVCs) in the specified Kubernetes namespace. Helps administrators track storage volume claims and verify that pods have the necessary persistent storage resources provisioned correctly.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 57, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources QPS", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC requests to TiKV grouped by their source components (e.g., TiDB, PD, other TiKV nodes). Helps identify which components are generating the most traffic and diagnose communication patterns or potential bottlenecks between cluster components.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 559, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL expire count", "group": "TTL", "used_for": "Used for monitoring the rate of key-value pairs being expired by TiKV's TTL (Time-To-Live) mechanism per instance. Helps track TTL cleanup activity and identify instances with high expiration rates, which may indicate data lifecycle issues or potential performance impact from TTL processing.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources duration", "group": "gRPC", "used_for": "Used for monitoring the duration of gRPC requests grouped by their source components in TiKV. Helps identify which upstream services (TiDB, PD, or other TiKV nodes) are generating the most gRPC traffic and diagnose communication bottlenecks between cluster components.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring TiDB pod CPU utilization in relation to Horizontal Pod Autoscaler (HPA) targets. Helps operators verify if auto-scaling is working correctly by comparing actual CPU usage against configured HPA thresholds, essential for diagnosing scaling behavior and resource allocation issues.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n", "(\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n /\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n) * 100" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 58, "dashboard": "Dataplane Resource Overview", "title": "Avg Mem Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring average memory utilization of TiDB pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether memory-based autoscaling is functioning correctly or if pods are under/over-provisioned.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "1", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"} and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 560, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL expire size", "group": "TTL", "used_for": "Used for monitoring the rate of key-value data size being expired and removed by TiKV's TTL (Time-To-Live) mechanism. Helps track TTL cleanup efficiency and identify instances with high expiration activity, which is useful for understanding storage reclamation patterns and potential performance impact from TTL operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC resource group QPS", "group": "gRPC", "used_for": "Used for monitoring the queries per second (QPS) handled by each resource group in TiKV's gRPC layer. Essential for tracking workload distribution across resource groups and identifying which groups are consuming the most gRPC resources, helping diagnose resource control effectiveness and potential bottlenecks in multi-tenant scenarios.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiDB worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for tracking auto-scaling behavior and identifying when worker pods are scaling up/down in response to workload changes or hitting configured capacity limits.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 564, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker poll interval", "group": "TTL", "used_for": "Used for monitoring the polling interval of TiKV's TTL (Time To Live) checker, which determines how frequently the garbage collection process checks for expired data. Helps diagnose GC scheduling issues and ensures the TTL checker is running at the expected frequency to maintain storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 563, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker compact duration", "group": "TTL", "used_for": "Used for monitoring the latency of TTL (Time To Live) checker compaction operations in TiKV, showing p99 and p9999 percentiles. Critical for diagnosing performance issues with expired data cleanup and identifying when TTL compaction operations are taking unusually long, which could impact storage efficiency and query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "Backup", "used_for": "Used for monitoring CPU utilization of TiKV backup worker threads during backup operations. Essential for identifying backup performance bottlenecks, detecting resource contention, and ensuring backup tasks don't overload the cluster.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 70, "dashboard": "Dataplane Resource Overview", "title": "HPA info(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of coprocessor worker pods, showing desired, current, and maximum replica counts. Essential for tracking autoscaling behavior and ensuring coprocessor workers scale appropriately to handle workload demands without hitting configured limits.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 63, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying when storage layer writes are experiencing delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 562, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker actions", "group": "TTL", "used_for": "Used for monitoring the rate of TTL (Time To Live) checker actions in TiKV, broken down by action type. Helps diagnose TTL-related I/O activity and identify issues with expired data cleanup operations, such as excessive deletion rates or stalled TTL processing.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the rate of different storage commands (get, scan, prewrite, commit, etc.) executed in TiKV. Essential for understanding storage operation patterns, identifying command distribution imbalances, and diagnosing transaction processing bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tidb_worker_hint(tidb-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiDB worker pods, comparing target scaling thresholds against current metric values. Helps operators understand when and why TiDB workers are being scaled up or down based on the custom `dxf_hscale_tidb_worker_hint` metric in Kubernetes environments.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"} *1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 35, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Thread Count", "group": "Backup", "used_for": "Used for monitoring the number of backup threads allocated in each TiKV instance's backup thread pool. Helps diagnose backup performance issues and resource allocation problems when backup operations are slow or consuming excessive resources.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 79, "dashboard": "Dataplane Resource Overview", "title": "HPA Evaluation Result(coprocessor-worker)", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the Horizontal Pod Autoscaler (HPA) evaluation results for coprocessor worker pods in the dataplane. Helps operators understand autoscaling decisions and verify whether the system is scaling coprocessor workers appropriately based on workload demands.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 61, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of coprocessor-worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for verifying HPA scaling behavior and determining if coprocessor workloads are triggering appropriate auto-scaling actions based on CPU pressure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 1.1", "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 0.9", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-coprocessor)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 64, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing write performance bottlenecks and identifying when storage layer async writes are experiencing high tail latency that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 66, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing slow snapshot generation issues that can impact read performance, backup operations, and follower read consistency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "Backup", "used_for": "Used for monitoring the rate at which SST (Sorted String Table) files are generated during backup operations in TiKV. Helps assess backup performance and identify throughput bottlenecks when creating backup snapshots.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 67, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations during pure local reads in TiKV's storage engine. Critical for diagnosing performance issues with local read operations that don't require cross-region data access or Raft consensus.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "Dataplane Resource Overview", "title": "Scaling Evnt Detials(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaling behavior of coprocessor workers, tracking current replicas, desired replicas, and the scaling gap. Essential for diagnosing autoscaling responsiveness issues and determining whether coprocessor resources are scaling appropriately to meet workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "last_over_time(kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])", "last_over_time(kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])\n", "(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n -\n kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n) != 0", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval])\n", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset -15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset 15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval])", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset -15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset 15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval])\n", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset -15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset 15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"}[$__interval:$__interval])\n/\nlast_over_time(kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}[$__interval:$__interval])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 10, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Errors", "group": "Backup", "used_for": "Used for tracking the rate of backup operation errors in TiKV instances. Essential for quickly identifying backup failures and diagnosing issues during data backup processes, helping ensure backup reliability and data protection.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 69, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tikv-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiKV worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for tracking auto-scaling behavior and identifying when TiKV workers are scaling up/down in response to load changes or hitting configured capacity limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 37, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "Backup", "used_for": "Used for monitoring the size distribution of SST files being backed up from TiKV's Write Column Family. Helps identify backup I/O patterns and diagnose issues with backup data volume or performance bottlenecks during backup operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 69, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the time spent waiting for read index proposals in TiKV's storage engine during snapshot reads. Critical for diagnosing read latency issues caused by Raft read index synchronization delays, especially when investigating slow queries or read performance degradation in strongly consistent read scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiKV, broken down by error status type. Critical for identifying storage engine issues, detecting data access failures, and diagnosing underlying problems with RocksDB operations or disk I/O errors.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Snapshot Duration", "group": "Backup", "used_for": "Used for monitoring the duration of backup snapshot operations in TiKV. Helps diagnose slow backup performance and identify bottlenecks during cluster backup processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 65, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine. Critical for diagnosing performance issues with read operations that require consistent snapshots, such as point queries and range scans.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 16, "dashboard": "Dedicated Dataplane Node Info", "title": "Node Info", "group": "", "used_for": "Used for displaying basic node information in Kubernetes clusters, including node names, kernel versions, and OS details. Helps operators quickly identify node characteristics and verify cluster configuration when troubleshooting infrastructure-level issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "instance", "k8s_cluster" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node,k8s_cluster_info) group_right kube_node_info" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 39, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "Backup", "used_for": "Used for monitoring the latency of saving SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance bottlenecks and identifying slow file write operations that could impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 73, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Process Stat Cpu Usage", "group": "Storage", "used_for": "Used for monitoring CPU usage of TiKV storage processes at the instance level. Essential for identifying CPU bottlenecks, resource contention, and determining if storage operations are CPU-bound, helping diagnose performance degradation and capacity planning needs.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "Dataplane Resource Overview", "title": "Scaling Event Distribution(coprocessor-worker)", "group": "", "used_for": "Used for tracking horizontal pod autoscaler (HPA) scaling events for coprocessor workers in the dataplane. Helps identify when and how frequently the coprocessor worker pool scales up or down in response to workload changes, essential for understanding autoscaling behavior and capacity management.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace" ], "promql": [ "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} > bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])", "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} < bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 60, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tikv-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of TiKV worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether CPU-based autoscaling is functioning correctly to handle workload changes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\", \n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-default)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 43, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup", "used_for": "Used for monitoring the p99 latency of external storage connection initialization during backup operations in TiKV. Critical for diagnosing slow backup starts or connection issues with external storage systems like S3, GCS, or Azure Blob Storage.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 69, "dashboard": "Dedicated Dataplane Node Info", "title": "Notice", "group": "", "used_for": "**Used For:**\n\nThis is an informational panel that displays important notices, warnings, or configuration guidance for the Dedicated Dataplane Node dashboard. Users should check this panel first to understand any critical information, limitations, or prerequisites before interpreting other monitoring metrics on this dashboard.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 70, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index propose wait operations in TiKV's storage engine, tracking p99 and p9999 percentiles. Critical for diagnosing delays in the Raft read index proposal phase, which can indicate leader election issues, network problems, or Raft consensus bottlenecks affecting read performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 71, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's storage engine. Critical for diagnosing performance issues with snapshot reads that require leader confirmation, particularly when investigating slow read queries or Raft read index bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tikv_worker_hint(tikv-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiKV worker pods, comparing target thresholds against current metric values. Helps operators understand autoscaling behavior and diagnose whether TiKV workers are scaling appropriately based on the custom `dxf_hscale_tikv_worker_hint_v2` metric.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 14, "dashboard": "Dedicated Dataplane Node Info", "title": "Node labels", "group": "", "used_for": "Used for displaying Kubernetes node labels and metadata for dedicated dataplane nodes. Helps operators verify node configurations, identify node roles, and troubleshoot node-specific issues by correlating system metrics with Kubernetes label assignments.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "2", "instance", "k8s_cluster", "namespace", "pod" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node, k8s_cluster_info) group_right sum(kube_node_labels) by (node, k8s_cluster_info, label_zone, label_dedicated, label_failure_domain_beta_kubernetes_io_zone, label_beta_kubernetes_io_instance_type)", "(\n label_replace(\n node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"},\n \"node\", \"$0\", \"instance\", \".*\"\n )\n * on(node, k8s_cluster_info) group_right\n sum(kube_node_labels)\n by (node, k8s_cluster_info,\n label_zone, label_dedicated,\n label_failure_domain_beta_kubernetes_io_zone,\n label_beta_kubernetes_io_instance_type)\n)\n* on(node) group_left(pod, component)\n label_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\"$pod\"},\n \"component\", \"$2\", \"pod\",\n \"(^|.*-)(tidb|tikv|tiflash|pd|tso|ticdc)(-[0-9a-z]+)*$\"\n )" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 41, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup", "used_for": "Used for monitoring the latency distribution when creating connections to external storage systems (like S3, GCS) during backup operations in TiKV. Critical for diagnosing slow backup initialization issues and external storage connectivity problems that could delay backup job startup.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 13, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "Backup", "used_for": "Used for monitoring disk I/O utilization percentage on cluster nodes during backup operations. Helps identify I/O bottlenecks and determine if disk saturation is impacting backup performance or causing resource contention with the TiDB cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 12, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Backup", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze requests during backup operations in TiKV. Critical for diagnosing performance bottlenecks when backup jobs perform data validation or when ANALYZE operations collect table statistics.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 38, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "Backup", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in the default column family during backup operations. Helps identify backup performance issues and assess the data volume being backed up from TiKV's default storage.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 72, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's Raft consensus protocol, tracking p99 and p9999 percentiles. Critical for diagnosing read performance issues related to Raft leader lease verification and identifying when follower reads experience delays in confirming the read index with the leader.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 73, "dashboard": "Dataplane Resource Overview", "title": "Scale Out summay[WIP](coprocessor-worker)", "group": "", "used_for": "Used for monitoring horizontal pod autoscaler (HPA) activity and scaling events for TiDB coprocessor workers in Kubernetes environments. Helps track when and how frequently coprocessor workers scale out based on CPU utilization, enabling diagnosis of autoscaling behavior and capacity planning issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", " kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1", " delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n <=\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) <= 0\n )\n )[$__range:15s]\n))" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 10, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received", "group": "", "used_for": "Used for monitoring the network ingress traffic rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual traffic patterns, or potential network bottlenecks affecting node communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 6, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Duration", "group": "Backup", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, showing p95, p99, and average durations. Critical for diagnosing backup performance issues and identifying slow backup ranges that may impact backup completion time or cluster performance during backup operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 74, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Full compaction duration seconds", "group": "Storage", "used_for": "Used for monitoring the latency of full compaction operations in TiKV's storage engine, tracking p99 and p9999 percentiles. Critical for diagnosing storage performance degradation and identifying when background compaction processes are taking unusually long, which can impact overall cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "Dedicated Dataplane Node Info", "title": "Component Node Number Trends", "group": "", "used_for": "Used for tracking the number of active nodes for each TiDB component (TiDB, TiKV, TiFlash) over time in a dedicated dataplane deployment. Helps monitor cluster scaling events, detect unexpected node failures, and verify that the expected number of component instances are running.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "3", "namespace" ], "promql": [ "count(\n label_replace(\n kube_pod_info{namespace=\"$namespace\", pod=~\"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"},\n \"component\", \n \"$1$3\", \n \"pod\", \n \"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"\n )\n) by (component)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 75, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Full compaction pause duration", "group": "Storage", "used_for": "Used for monitoring the pause duration during full compaction operations in TiKV's storage engine, tracking both p99 and p9999 latencies. Critical for diagnosing storage performance degradation and identifying when full compaction causes excessive pauses that may impact write availability or query responsiveness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 76, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Full compaction per-increment duration", "group": "Storage", "used_for": "Used for monitoring the duration of incremental full compaction operations in TiKV's storage engine at p99 and p9999 percentiles. Critical for diagnosing storage layer performance issues and identifying when background compaction processes are taking unusually long, which can impact overall write performance and storage efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted", "group": "", "used_for": "Used for monitoring the network transmission rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual outbound traffic patterns, or potential network bottlenecks affecting data replication and client communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 55, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import CPU Utilization", "group": "", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import operations. Essential for diagnosing performance bottlenecks during backup restoration or bulk data loading, helping identify whether import processes are CPU-bound or if thread resources are properly utilized.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 68, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations during pure local reads in TiKV's storage engine, tracking p99 and p9999 percentiles. Critical for diagnosing read performance issues when data is served directly from local storage without requiring Raft consensus, helping identify storage layer bottlenecks in read-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets on network interfaces during packet reception on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that cause packet loss and degrade cluster communication performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 47, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted Drop", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of dropped packets during network transmission on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that prevent packets from being sent successfully.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 78, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Local reader requests", "group": "Local Reader", "used_for": "Used for monitoring the rate of local read requests in TiKV's raftstore, including successfully executed reads, stale reads, and rejected requests with their rejection reasons. Essential for diagnosing local reader performance issues, understanding read request patterns, and identifying why certain reads cannot be served locally (requiring fallback to Raft consensus reads).", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying slow scan operations that may impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 27, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Errors", "group": "", "used_for": "Used for monitoring the rate of import errors occurring in TiKV during data import operations. Essential for quickly identifying and diagnosing import failures or issues when restoring backups or bulk loading data into the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 54, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_in_allowance_exceeded", "group": "", "used_for": "Used for monitoring network bandwidth ingress throttling events on dedicated dataplane nodes. Helps identify when incoming network traffic exceeds allocated bandwidth allowances, which can cause packet drops and performance degradation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_in_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 49, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import RPC Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of import RPC requests in TiKV during data import operations. Critical for diagnosing performance bottlenecks and slow import issues when restoring backups or bulk loading data into the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 56, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Thread Count", "group": "", "used_for": "Used for monitoring the number of active SST import threads in TiKV instances during data import operations. Helps diagnose import performance issues and verify that import parallelism is properly configured when restoring backups or importing large datasets.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 55, "dashboard": "Dedicated Dataplane Node Info", "title": "linklocal_allowance_exceeded", "group": "", "used_for": "Used for monitoring the rate of link-local address allowance exceeded events on network interfaces in the dataplane node. Helps identify network configuration issues or excessive link-local traffic that may indicate connectivity problems or misconfigurations in the Kubernetes cluster networking layer.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_linklocal_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 82, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store writer CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's store writer threads, which handle data persistence to RocksDB. Essential for diagnosing write performance bottlenecks and identifying instances where store writer threads are saturated, potentially causing write stalls or increased latency.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 83, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC polling threads in TiKV instances. Essential for diagnosing network communication overhead and identifying instances with excessive gRPC processing load that may impact overall performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's Raft store threads, which handle Raft consensus operations and region management. Critical for diagnosing performance bottlenecks related to Raft log processing, region splits/merges, and identifying instances with excessive Raft workload that may impact cluster stability.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "Dedicated Dataplane Node Info", "title": "PPS_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring packet-per-second (PPS) rate limiting violations on network interfaces. Helps identify when network traffic exceeds configured PPS allowances, which can indicate traffic bursts, DDoS attempts, or misconfigured rate limits that may impact cluster network performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_pps_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 84, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's scheduler worker threads, which handle transaction scheduling and conflict detection. Essential for diagnosing transaction processing bottlenecks and identifying instances with high scheduling overhead that may impact write performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Wait Duration", "group": "", "used_for": "Used for monitoring the wait time in the import queue during data import operations in TiKV. Helps diagnose import bottlenecks and queue congestion when backup restoration or bulk data loading is slower than expected.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 19, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes or indicate disk saturation issues affecting cluster performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 46, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, helping identify whether slow imports are caused by write or download operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 85, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's unified read pool threads across instances. Essential for diagnosing read performance bottlenecks and identifying instances with high CPU consumption during read-heavy workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_out_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring network bandwidth throttling events when outbound traffic exceeds allocated limits on dedicated dataplane nodes. Critical for diagnosing network performance degradation caused by bandwidth quota exhaustion and identifying nodes that may need increased network capacity allocation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_out_allowance_exceeded{cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 99, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Table Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the connection tracking table utilization ratio on dataplane nodes. Critical for identifying when the conntrack table is approaching capacity, which can cause connection drops and network issues when the limit is reached.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "node_nf_conntrack_entries{cluster=\"$k8s_cluster\",instance=~\"$instance\"} / node_nf_conntrack_entries_limit{cluster=\"$k8s_cluster\",instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 57, "dashboard": "Dedicated Dataplane Node Info", "title": "conntrack_allowance_exceeded", "group": "", "used_for": "Used for monitoring connection tracking table overflow events on dedicated dataplane nodes. Helps diagnose network connection limits being exceeded, which can cause dropped connections and connectivity issues when the system attempts to track more connections than allowed.", "component": "", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_conntrack_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 86, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiKV instances. Essential for identifying storage engine CPU bottlenecks and diagnosing performance issues related to data compaction, flushing, and other RocksDB background operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 98, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Drops", "group": "", "used_for": "Used for monitoring the rate of dropped connection tracking entries on dedicated dataplane nodes. Critical for diagnosing network connection issues caused by conntrack table exhaustion, which can lead to packet drops and connection failures under high connection throughput scenarios.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 52, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Read SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file read operations during data import in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the SST file download/read phase of backup restoration or data migration processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 100, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Insert Failures", "group": "", "used_for": "Used for monitoring the rate of connection tracking table insertion failures on dataplane nodes. Critical for diagnosing network connection issues caused by conntrack table exhaustion, which can lead to dropped connections and increased latency when the system cannot track new network flows.", "component": "", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_insert_failed{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 90, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV background worker threads across instances. Essential for diagnosing performance issues related to background tasks like compaction, cleanup, and maintenance operations that may compete with foreground workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "**Used For:**\n\nUsed for monitoring the current memory utilization on dedicated dataplane nodes. Essential for identifying memory pressure, potential OOM risks, and capacity planning for node resources.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(\n (\n (\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n )\n / node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n ) * 100)\n" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 53, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets through tools like Lightning or BR restore operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 47, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "", "used_for": "Used for monitoring the latency distribution of ingest RPC requests during data import operations in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets, helping identify slow ingest operations that may impact overall import throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 101, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Early Drops", "group": "", "used_for": "Used for monitoring the rate of connection tracking table entries that are dropped prematurely due to table exhaustion on dataplane nodes. Critical for diagnosing network connection limits and identifying when the conntrack table is full, which can cause connection failures and packet drops under high connection throughput scenarios.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_early_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 88, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's region worker threads, which handle region-related operations like split, merge, and scheduling tasks. Essential for diagnosing region management performance bottlenecks and identifying instances with excessive region operation overhead.", "component": "TIKV", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"region_worker.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "", "used_for": "Used for monitoring the rate of SST (Sorted String Table) file ingestion during data import operations in TiKV. Critical for tracking import throughput and diagnosing slow data loading performance during backup restoration or bulk data import scenarios.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 50, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "", "used_for": "Used for monitoring the latency of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data, particularly when SST files need to be rewritten to match the target cluster's key ranges.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 2, "dashboard": "Dedicated Dataplane Node Info", "title": "System load", "group": "CPU", "used_for": "Used for monitoring the system load averages (1, 5, and 15 minutes) on dedicated dataplane nodes. Essential for identifying CPU resource saturation and determining if the node is overloaded or experiencing sustained high workload pressure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(node_load1{job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load5{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load15{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 31, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during data import operations in TiKV. Helps diagnose import performance bottlenecks and network bandwidth issues when restoring backups or importing large datasets.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 3, "dashboard": "Dedicated Dataplane Node Info", "title": "Usage Per Core", "group": "CPU", "used_for": "Used for monitoring CPU utilization per individual core on dedicated dataplane nodes. Helps identify CPU core imbalances, hotspots, and whether workloads are properly distributed across all available cores.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "sum by (cpu) (irate(node_cpu_seconds_total{ job=\"node-exporter\", mode!=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m]))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 93, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV backup-related threads (backup-worker, backup_endpoint). Essential for diagnosing performance issues during backup operations and identifying whether backup tasks are consuming excessive CPU resources that might impact cluster performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 87, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's garbage collection (GC) worker threads across instances. Essential for diagnosing GC performance issues, identifying instances with excessive GC overhead, and detecting potential bottlenecks in MVCC version cleanup operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 29, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Region", "group": "", "used_for": "Used for monitoring the total number of Raft regions on each TiKV instance during backup and import operations. Helps identify region distribution imbalances and potential hotspots that could impact backup/restore performance or cause cluster instability.", "component": "", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 4, "dashboard": "Dedicated Dataplane Node Info", "title": "CPU Utilization", "group": "CPU", "used_for": "Used for monitoring the overall CPU utilization percentage on dedicated dataplane nodes. Essential for identifying CPU bottlenecks, resource saturation, and determining if nodes need scaling or workload rebalancing.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m])) * 100)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 6, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "Used for monitoring memory consumption on dedicated dataplane nodes, showing total used memory, buffer cache, and page cache separately. Essential for identifying memory pressure, detecting potential OOM conditions, and understanding how the system allocates memory between applications and kernel caches.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "node", "port" ], "promql": [ "max(\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n)\n", "max(node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$node:$port\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 25, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Leader", "group": "", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance and detecting sudden leader drops. Critical for identifying leader balance issues, region scheduling problems, or potential TiKV node failures during backup and import operations when leader count drops by more than 10 in a 2-minute window.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 100, "dashboard": "PD-Cluster-Clinic", "title": "Storage capacity", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity available across the entire TiDB cluster. Essential for capacity planning and identifying when the cluster is approaching storage limits that could impact write operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 89, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snap generator CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's snapshot generator threads, which create snapshots for Raft log replication and region recovery. Essential for diagnosing performance issues related to snapshot generation overhead, especially during region splits, leader transfers, or when adding new replicas.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"snap_generator.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes and determining if disk subsystems are saturated.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 102, "dashboard": "PD-Cluster-Clinic", "title": "Current storage used", "group": "", "used_for": "Used for monitoring the current storage utilization ratio across the TiDB cluster. Helps identify when the cluster is approaching storage capacity limits and needs scaling or data cleanup to prevent storage exhaustion issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 57, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze coprocessor requests in TiKV during backup and import operations. Critical for diagnosing performance bottlenecks when TiDB performs table checksums for data validation or statistics collection during backup/restore workflows.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 101, "dashboard": "PD-Cluster-Clinic", "title": "Current storage size", "group": "", "used_for": "Used for monitoring the total storage capacity currently used across the entire TiDB cluster. Helps track storage growth trends and plan capacity expansion before running out of disk space.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 45, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk I/O Util", "group": "Disk", "used_for": "Used for monitoring disk I/O utilization percentage on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting saturated storage devices, and diagnosing performance degradation caused by excessive I/O operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 91, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftlog fetch worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's raftlog fetch worker threads, which handle fetching Raft logs from the log storage engine. Essential for diagnosing performance bottlenecks when TiKV nodes are reading Raft logs, particularly useful when investigating replication lag or slow follower read issues.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 92, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's SST (Sorted String Table) import threads, which handle data ingestion during operations like Lightning imports or snapshot restoration. Essential for diagnosing performance bottlenecks during bulk data import operations and identifying instances with high import workload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 103, "dashboard": "PD-Cluster-Clinic", "title": "Normal stores", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total number of healthy TiKV stores currently online in the PD cluster. Essential for quickly assessing cluster health and detecting store failures or unavailability that could impact data availability and performance.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_up_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 95, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TSO worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TSO (Timestamp Oracle) worker threads in TiKV instances. Essential for diagnosing performance bottlenecks in timestamp allocation, which can impact transaction processing speed and overall cluster throughput when CPU usage is abnormally high.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk iops", "group": "Disk", "used_for": "Used for monitoring disk I/O operations per second (IOPS) on dedicated dataplane nodes, tracking both read and write operations. Essential for identifying disk bottlenecks, diagnosing storage performance issues, and determining if workload demands exceed disk capacity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 94, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CDC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of CDC (Change Data Capture) worker threads in TiKV instances, including cdcwkr threads, TSO threads, and other CDC-related threads. Essential for diagnosing CDC replication performance issues, identifying CPU bottlenecks in change data capture workloads, and ensuring sufficient resources for real-time data synchronization tasks.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "PD-Cluster-Clinic", "title": "Current peer count", "group": "", "used_for": "Used for monitoring the total number of Raft peer replicas across all regions in the PD cluster. Essential for verifying cluster health, detecting replica imbalances, and ensuring data redundancy meets configured requirements.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 91, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk iops", "group": "Disk", "used_for": "Used for monitoring write I/O operations per second on disk devices for dedicated dataplane nodes. Helps identify disk write throughput bottlenecks and diagnose storage performance issues when write-heavy workloads cause system slowdowns.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 22, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average wait time per disk I/O operation (both reads and writes) on dedicated dataplane nodes. Essential for identifying disk performance bottlenecks and diagnosing storage-related latency issues that may impact TiDB cluster performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 97, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's coprocessor read pool threads across different priority levels (normal, high, low). Essential for diagnosing coprocessor request performance issues, identifying CPU bottlenecks in analytical query processing, and detecting imbalanced workload distribution across TiKV instances.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's async apply threads, which handle applying Raft log entries to the RocksDB storage engine. Essential for diagnosing write performance bottlenecks and identifying instances where apply threads are saturated, which can cause write stalls and increased latency.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 104, "dashboard": "PD-Cluster-Clinic", "title": "Number of Regions", "group": "", "used_for": "Used for monitoring the total number of leader regions across the PD cluster. Essential for tracking cluster data distribution and identifying potential region imbalances or abnormal region count changes that may indicate scheduling issues or cluster scaling events.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"leader_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 98, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IME CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's In-Memory Engine (IME) threads across instances. Essential for diagnosing performance bottlenecks and resource contention in the in-memory storage layer, particularly when investigating memory-intensive workload issues or IME-related performance degradation.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average time spent waiting for disk write operations to complete on dataplane nodes. Essential for identifying disk I/O bottlenecks and diagnosing storage performance degradation that could impact TiKV write throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 99, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Busy Threads (>80%)", "group": "Thread CPU", "used_for": "Used for identifying the top 20 most CPU-intensive threads in TiKV instances that are consuming over 80% CPU. Critical for diagnosing performance bottlenecks and pinpointing which specific threads (excluding RocksDB threads) are causing high CPU utilization during performance degradation.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 96, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether storage read threads are saturated or unevenly loaded across instances.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "PD-Cluster-Clinic", "title": "Abnormal stores", "group": "", "used_for": "Used for monitoring the health status of TiKV stores in the cluster by tracking disconnected, unhealthy, and low-space stores. Critical for quickly identifying storage node failures, network issues, or capacity problems that could impact cluster availability and data reliability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_disconnected_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_unhealth_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_low_space_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_down_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_offline_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_tombstone_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_slow_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 102, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of read I/O bytes across TiKV instances, broken down by I/O type (e.g., foreground, background operations). Essential for diagnosing read performance bottlenecks, identifying I/O-intensive workloads, and understanding storage layer read throughput patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring disk I/O bandwidth (read and write rates) on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting abnormal I/O patterns, and diagnosing performance issues related to storage throughput in the cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 103, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO threshold", "group": "IO Breakdown", "used_for": "Used for monitoring the configured I/O rate limit thresholds in TiKV's rate limiter by operation type. Helps administrators verify I/O throttling settings and diagnose whether performance issues are caused by hitting configured rate limits.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine, tracking both P99 and P9999 percentiles. Critical for diagnosing write performance bottlenecks and identifying tail latency issues in the storage layer that may impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 104, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate Limiter Request Wait Duration", "group": "IO Breakdown", "used_for": "Used for monitoring the 99th percentile wait time for requests throttled by TiKV's rate limiter. Critical for diagnosing performance degradation caused by I/O throttling and identifying when storage bandwidth limits are impacting request latency.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 101, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of write I/O throughput in bytes per second across TiKV instances, broken down by I/O type (e.g., data, WAL, compaction). Essential for diagnosing write performance bottlenecks, identifying which storage operations consume the most I/O bandwidth, and detecting abnormal disk write patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 93, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring the write bandwidth (bytes per second) to physical disks on dedicated dataplane nodes. Essential for identifying disk I/O bottlenecks, detecting write-heavy workloads, and diagnosing storage performance issues that may impact TiKV write operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 110, "dashboard": "PD-Cluster-Clinic", "title": "Online store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store preparation operations in PD, specifically tracking stores that are in the \"preparing\" state before coming online. Helps diagnose slow store startup issues and identify when new TiKV nodes are taking longer than expected to join the cluster.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"preparing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 32, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of individual disk read and write operations per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and optimizing disk utilization.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 92, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of disk I/O operations (both read and write) per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and potential bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 109, "dashboard": "PD-Cluster-Clinic", "title": "Offline store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store removal operations in PD when taking TiKV nodes offline. Helps track decommissioning status and estimate completion time when scaling down the cluster or replacing failed nodes.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"removing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 112, "dashboard": "PD-Cluster-Clinic", "title": "Current scaling speed", "group": "", "used_for": "Used for monitoring the current rate at which PD is scaling the cluster, measured in operations per minute. Helps identify whether cluster rebalancing and region scheduling are proceeding at expected speeds or if scaling operations are throttled or stalled.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 48, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the percentage of used storage across different devices. Essential for preventing disk full scenarios and identifying nodes approaching storage capacity limits that could impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "((max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) /\n max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) * 100" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 30, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read (rrqm) and write (wrqm) requests per second on disk devices in the dataplane node. Helps assess disk I/O efficiency and identify whether the kernel is successfully merging adjacent I/O requests to optimize disk performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 95, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read and write requests per second on disk devices in dedicated dataplane nodes. Helps identify disk I/O optimization efficiency and potential bottlenecks when the OS merges adjacent I/O requests to improve throughput.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 111, "dashboard": "PD-Cluster-Clinic", "title": "Left time", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the estimated time remaining for PD cluster operations such as region scheduling, leader balancing, or other ongoing cluster adjustments. Helps operators understand when current rebalancing or migration tasks are expected to complete, enabling better planning for maintenance windows and capacity changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 107, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft store thread operations in TiKV, tracking p99 and p9999 percentiles. Critical for diagnosing Raft log processing bottlenecks and identifying when the store thread becomes a performance bottleneck in the write path.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 203, "dashboard": "PD-Cluster-Clinic", "title": "Cluster ID", "group": "Cluster", "used_for": "Used for displaying the unique cluster identifier of the PD (Placement Driver) cluster. Helps verify cluster identity and distinguish between multiple TiDB clusters in multi-cluster environments or during troubleshooting.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 108, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of applying Raft log entries to TiKV's state machine, tracking p99 and p9999 percentiles. Critical for diagnosing write path performance issues and identifying bottlenecks in the Raft consensus apply stage that can impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization of TiKV and TiFlash persistent volume claims in Kubernetes environments. Essential for tracking storage consumption and preventing disk full issues that could cause data write failures or service disruptions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 51, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiKV and TiFlash persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized deployments.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 201, "dashboard": "PD-Cluster-Clinic", "title": "PD scheduler config", "group": "Cluster", "used_for": "Used for monitoring the configuration status of PD schedulers in the cluster. Helps verify scheduler settings and diagnose issues related to region balancing, leader distribution, and other scheduling behaviors that may affect cluster performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 113, "dashboard": "PD-Cluster-Clinic", "title": "Placement Rules Status", "group": "", "used_for": "Used for monitoring the status of PD's placement rule manager, which controls data placement policies across the TiKV cluster. Essential for verifying that placement rules are properly configured and active, helping diagnose issues with data distribution, replica placement, and label-based scheduling constraints.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_rule_manager_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 97, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiDB persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized TiDB deployments.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 112, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send proposal duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft proposals from TiKV store threads to Raft threads, tracking p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when the store thread becomes a performance bottleneck in the Raft replication pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 202, "dashboard": "PD-Cluster-Clinic", "title": "Store limit", "group": "Cluster", "used_for": "Used for monitoring the configured rate limits on TiKV stores, which control the maximum number of operations (like adding/removing peers) allowed per store. Essential for diagnosing scheduling bottlenecks and understanding why region rebalancing or recovery might be slower than expected.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [], "promql": [ "pd_cluster_store_limit" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 205, "dashboard": "PD-Cluster-Clinic", "title": "Label distribution", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of placement rule labels across the PD cluster. Helps diagnose data placement issues, verify label configuration correctness, and ensure regions are properly distributed according to placement policies for high availability and disaster recovery.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (name)", "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 96, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption of TiDB Persistent Volume Claims (PVCs) in Kubernetes environments. Essential for tracking storage utilization and preventing disk space exhaustion issues that could impact TiDB server operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 49, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Used", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the difference between total filesystem size and available space. Essential for identifying storage capacity issues and preventing disk full scenarios that could impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "(max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 8, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Helps identify read-heavy workloads, diagnose storage performance bottlenecks, and verify if EBS volume IOPS limits are being reached during high read activity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 109, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store propose wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft proposals spend waiting in the store thread queue before being processed in TiKV. Critical for diagnosing write latency issues caused by store thread congestion or scheduling delays, especially when p99/p9999 latencies spike, indicating the store thread is becoming a bottleneck in the Raft replication pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 114, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store before write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the duration before Raft store writes are executed in TiKV, tracking p99 and p9999 latencies. Critical for diagnosing Raft log replication bottlenecks and identifying delays in the pre-write phase of the Raft consensus process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 108, "dashboard": "PD-Cluster-Clinic", "title": "Leader/Primary", "group": "", "used_for": "Used for identifying the current leader/primary member in the PD (Placement Driver) cluster. Essential for verifying cluster leadership status, troubleshooting split-brain scenarios, and confirming which PD instance is actively managing cluster metadata and scheduling decisions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "service_member_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 110, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store batch wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft store threads spend waiting to batch process requests in TiKV, tracking both p99 and p9999 latencies. Critical for diagnosing Raft processing bottlenecks and identifying when store threads are experiencing delays before batch operations, which can indicate thread contention or scheduling issues in the Raft store layer.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 204, "dashboard": "PD-Cluster-Clinic", "title": "Region label isolation level", "group": "Cluster", "used_for": "Used for monitoring the distribution of regions across different label isolation levels in PD's placement rules. Helps verify that regions are properly distributed according to configured label constraints (e.g., rack, zone, host) for high availability and fault tolerance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 64, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring write input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Essential for identifying storage bottlenecks, evaluating whether workloads are approaching EBS IOPS limits, and diagnosing write-heavy performance issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 111, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send to write queue duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft store thread messages to the write queue in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when the Raft store thread experiences delays in queuing write operations, which can indicate thread contention or queue saturation issues.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance", "used_for": "Used for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits on dedicated dataplane nodes. Critical for identifying storage bottlenecks caused by insufficient EBS volume capacity or EC2 instance bandwidth limits that impact TiDB cluster performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 210, "dashboard": "PD-Cluster-Clinic", "title": "Goroutine Count", "group": "Cluster", "used_for": "Used for monitoring the number of active goroutines in PD, TSO, and scheduling components. Helps identify goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate deadlocks or performance degradation in the PD cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 206, "dashboard": "PD-Cluster-Clinic", "title": "Current ID allocation", "group": "Cluster", "used_for": "Used for monitoring the current ID allocation counter in PD's cluster ID allocator. Helps track ID generation progress and diagnose issues with cluster-wide unique ID assignment for transactions, regions, and other distributed components.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"idalloc\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 115, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of persisting Raft store metadata to disk in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when disk I/O issues are causing Raft store operations to slow down, which can impact overall cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 208, "dashboard": "PD-Cluster-Clinic", "title": "Memory Usage", "group": "Cluster", "used_for": "Used for monitoring memory consumption of PD (Placement Driver) components including resident memory, heap system memory, and heap in-use memory. Essential for detecting memory leaks, identifying OOM risks, and diagnosing PD performance degradation caused by excessive memory usage.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}", "pd_service_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 207, "dashboard": "PD-Cluster-Clinic", "title": "CPU Usage", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of PD (Placement Driver) components including PD, TSO, and scheduling services. Essential for identifying CPU bottlenecks, resource contention, and determining if PD nodes are overloaded, which can impact cluster metadata management and timestamp allocation performance.", "component": "PD", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 117, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit but not persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time gap between when Raft logs are committed and when they are actually persisted to disk in TiKV stores. Critical for diagnosing write pipeline bottlenecks and identifying delays in the fsync/flush operations that could impact data durability and write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 116, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of the store write end phase in TiKV's Raft replication waterfall, tracking p99 and p9999 percentiles. Critical for diagnosing write performance bottlenecks in the final stage of Raft log persistence, helping identify when store-level write operations are experiencing tail latency issues.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum read I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume read operations are experiencing high latency that could impact TiKV read performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 212, "dashboard": "PD-Cluster-Clinic", "title": "Max TSO gap", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the maximum TSO (Timestamp Oracle) gap in milliseconds between different data centers in a PD cluster. Critical for identifying clock synchronization issues and potential data consistency problems in multi-DC deployments, where large TSO gaps can indicate network latency or PD service degradation.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 211, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Physcial", "group": "Cluster", "used_for": "Used for monitoring the current physical timestamp value of PD's global TSO (Timestamp Oracle). Essential for verifying TSO service health, detecting time regression issues, and ensuring monotonic timestamp allocation across the distributed cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 113, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write kv db end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of completing KV database writes in TiKV's raftstore, tracking p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when data persistence to RocksDB is experiencing delays in the Raft replication waterfall.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 67, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify I/O bottlenecks, diagnose storage performance issues, and verify whether read-intensive workloads are approaching EBS volume throughput limits.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 66, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum write I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume write operations are experiencing delays that could impact TiKV write performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 209, "dashboard": "PD-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring how long each PD (Placement Driver) component instance has been running since its last restart. Essential for tracking cluster stability, identifying recent restarts or crashes, and correlating service disruptions with component availability issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 65, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the write throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify write bandwidth bottlenecks, verify if workloads are approaching EBS volume throughput limits, and diagnose I/O performance issues in cloud storage.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 120, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Process ready duration", "group": "Raft IO", "used_for": "Used for monitoring the time spent processing ready Raft entries in TiKV's Raft store. Critical for diagnosing Raft replication delays and identifying bottlenecks in the consensus protocol's ready state handling, which can impact write latency and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 118, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit and persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft log commit and persist operations at the store level in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when disk I/O or Raft consensus operations are causing slow writes, especially useful when investigating tail latency issues in the Raft replication process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 122, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write loop duration", "group": "Raft IO", "used_for": "Used for monitoring the time spent in TiKV's store write loop, which handles persisting Raft log entries to disk. Critical for diagnosing write path bottlenecks and identifying when the store thread is overwhelmed, which can cause replication delays and impact overall cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 107, "dashboard": "PD-Cluster-Clinic", "title": "Region health", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the health status and distribution of regions across the TiDB cluster, categorized by type (e.g., healthy, down-peer, pending-peer, offline-peer). Essential for identifying region availability issues, replica problems, and cluster stability concerns that may impact data accessibility and performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 213, "dashboard": "PD-Cluster-Clinic", "title": "Scheduling Allowance Status", "group": "Cluster", "used_for": "Used for monitoring whether PD's scheduling operations are currently allowed or restricted in the cluster. Essential for diagnosing why region scheduling might be paused or limited, helping identify if scheduling is intentionally disabled due to maintenance, resource constraints, or automatic protection mechanisms.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 121, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Process ready duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft ready state processing per TiKV instance. Critical for diagnosing Raft IO performance bottlenecks and identifying instances experiencing delays in processing ready Raft proposals before they're applied to the state machine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 124, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency distribution of Raft log append operations in TiKV's Raftstore. Critical for diagnosing write path performance issues, as slow log appends directly impact transaction commit latency and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 214, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Datetime", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for displaying the current timestamp oracle (TSO) value in the PD cluster, which represents the global logical clock for distributed transactions. Essential for verifying TSO service availability and detecting time synchronization issues that could impact transaction ordering and consistency across the TiDB cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 72, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the average disk read wait latency on TiKV nodes at the EBS volume level. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk reads in cloud-deployed TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 302, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator check", "group": "Operator", "used_for": "Used for monitoring the rate at which PD checks different types of scheduling operators per minute. Helps diagnose scheduling decision-making efficiency and identify potential bottlenecks in PD's operator evaluation process.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"check\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 123, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write loop duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of TiKV's store write loop, which handles Raft log persistence to disk per server. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or Raft replication delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 90, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits at the cluster level. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to I/O throttling and degraded database performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 76, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait time for disk write operations on TiKV nodes' EBS volumes. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk writes at the infrastructure level.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 304, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator timeout", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that timeout, categorized by operator type. Critical for identifying scheduling bottlenecks and configuration issues when PD cannot complete region rebalancing or recovery operations within expected timeframes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"timeout\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 75, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB Cluster Pods", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for displaying all TiDB cluster pods running in the specified Kubernetes namespace. Helps operators quickly verify pod deployment status and identify which pods are active in the cluster for infrastructure monitoring and troubleshooting purposes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "namespace" ], "promql": [ "kube_pod_info{namespace=\"$namespace\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 126, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends committing Raft logs to disk. Critical for diagnosing write path performance issues, as high commit log duration directly impacts transaction commit latency and can indicate disk I/O bottlenecks or storage layer problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 303, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator finish", "group": "Operator", "used_for": "Used for monitoring the rate at which PD scheduling operators complete their operations, broken down by operator type. Helps track scheduling activity efficiency and identify which types of data rebalancing or region management operations are being executed most frequently.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"finish\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 79, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk wait latency", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait latency per disk I/O operation on EBS volumes in dedicated dataplane nodes. Essential for identifying storage performance bottlenecks and diagnosing whether slow query performance is caused by underlying disk I/O delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)", "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 301, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator create", "group": "Operator", "used_for": "Used for monitoring the rate at which PD creates different types of scheduling operators (e.g., balance-region, transfer-leader). Essential for understanding cluster rebalancing activity and diagnosing whether PD is actively responding to scheduling needs or if operator creation is blocked.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 78, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average disk write wait latency for TiFlash nodes at the cluster level. Helps identify storage I/O bottlenecks and disk performance degradation that could impact TiFlash's columnar storage operations and analytical query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(node)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 77, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average read wait latency of EBS volumes serving TiFlash nodes in dedicated dataplane deployments. Essential for identifying storage I/O bottlenecks and diagnosing slow query performance caused by disk read delays at the cluster level.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 125, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft log append operations on each TiKV instance. Critical for diagnosing write path performance issues, as high append log duration directly impacts transaction commit latency and can indicate disk I/O bottlenecks or Raft replication slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_append_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 130, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Client Wait Connection Ready Duration", "group": "Raft IO", "used_for": "Used for monitoring the time Raft clients spend waiting for connections to become ready before sending messages between TiKV nodes. Critical for diagnosing network connection establishment issues and Raft communication delays that can impact cluster consensus and replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 127, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft log commit operations per TiKV instance. Critical for diagnosing write path performance issues and identifying instances with slow disk I/O or Raft replication bottlenecks that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 305, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator replaced or canceled", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that are canceled or replaced, broken down by operator type. Essential for diagnosing scheduling instability, identifying conflicting scheduling decisions, and understanding why planned data movements are being interrupted or modified.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"cancel\"}[2m])*60) by (type)", "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"replace\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 128, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends applying Raft logs to the state machine. Critical for diagnosing write path performance issues, as high apply log duration indicates bottlenecks in persisting committed data to RocksDB, which directly impacts write latency and throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 306, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operators count by state", "group": "Operator", "used_for": "**Used For:**\n\nUsed for monitoring the rate of scheduling operator state changes in PD, categorized by event type (created, finished, timeout, etc.). Essential for diagnosing scheduling behavior, identifying stuck operators, and understanding whether the cluster is actively rebalancing or if scheduling operations are completing successfully.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 308, "dashboard": "PD-Cluster-Clinic", "title": "Operator step finish duration", "group": "Operator", "used_for": "Used for monitoring the P95 and P99 latency of PD operator step completion during cluster scheduling operations. Essential for diagnosing slow scheduling performance and identifying which operator types (e.g., region transfers, leader changes) are experiencing delays in execution.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 85, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP In Use", "group": "", "used_for": "Used for monitoring the number of TCP sockets currently in use on dedicated dataplane nodes. Helps identify socket exhaustion issues, connection leaks, or abnormal connection patterns that could impact network communication and service availability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "node_sockstat_TCP_inuse{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 307, "dashboard": "PD-Cluster-Clinic", "title": "Operator finish duration", "group": "Operator", "used_for": "Used for monitoring the p95 and p99 latency of PD operator completion times by operator type. Essential for diagnosing scheduling performance issues and identifying which operator types (like region merge, leader transfer, or replica addition) are experiencing delays in execution.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 87, "dashboard": "Dedicated Dataplane Node Info", "title": "Segments retransmitted", "group": "", "used_for": "Used for monitoring the rate of TCP segment retransmissions on dedicated dataplane nodes. Helps diagnose network reliability issues, packet loss, or congestion problems that may impact cluster communication and overall performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "rate(node_netstat_Tcp_RetransSegs{instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 401, "dashboard": "PD-Cluster-Clinic", "title": "Store capacity", "group": "Statistics - balance", "used_for": "Used for monitoring the total storage capacity of each TiKV store in the cluster. Essential for capacity planning and identifying stores approaching their storage limits before they impact cluster availability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_capacity\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 129, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of applying Raft logs to the state machine in TiKV instances. Critical for diagnosing write path performance issues and identifying instances experiencing slow log application, which can cause replication lag and affect overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 131, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Raft Client Wait Connection Ready Duration", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft clients waiting for connections to become ready between TiKV instances. Critical for diagnosing network connection establishment issues and inter-node communication delays that can impact Raft replication performance and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "connection", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) )", "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 310, "dashboard": "PD-Cluster-Clinic", "title": "99% operator region size", "group": "Operator", "used_for": "Used for monitoring the 99th percentile size of regions being processed by PD operators. Helps identify whether operator scheduling is handling abnormally large regions, which can impact rebalancing performance and cluster stability.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type,le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 133, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Write task block duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile duration that Raft write tasks are blocked waiting in the queue per TiKV server. Critical for diagnosing write performance bottlenecks and identifying when the Raft store thread is overloaded, causing write operations to experience excessive queueing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 398, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with source database changes in real-time.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 73, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Bandwidth", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring disk read and write bandwidth across EBS volumes in the dedicated dataplane cluster. Essential for identifying disk I/O bottlenecks, detecting abnormal storage traffic patterns, and ensuring EBS volumes are not approaching their throughput limits during heavy workloads.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])", "rate(node_disk_written_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 309, "dashboard": "PD-Cluster-Clinic", "title": "Schedulers reach limit", "group": "Operator", "used_for": "Used for monitoring when PD schedulers hit their configured limits, tracking both operator limits by type/name and store-level limit violations. Essential for diagnosing why region scheduling operations are being throttled or rejected, which can impact cluster rebalancing and recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (type,name)", "sum(rate(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (desc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 468, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring downstream systems receive near real-time updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 89, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP Connections", "group": "", "used_for": "Used for monitoring the current number of established TCP connections on dedicated dataplane nodes. Helps identify connection leaks, abnormal connection spikes, or resource exhaustion issues that could impact network communication and service availability.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance" ], "promql": [ "node_netstat_Tcp_CurrEstab{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 136, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft read and write proposals in TiKV, distinguishing between local reads, normal writes, and read index operations. Essential for diagnosing Raft consensus performance issues, identifying proposal bottlenecks, and understanding the distribution of read versus write workloads at the consensus layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft proposals spend waiting in queue before being processed in TiKV. Critical for diagnosing write latency issues caused by Raftstore thread saturation or proposal queue congestion.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 137, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft read proposals (local reads and read index requests) per TiKV instance. Helps identify read traffic distribution across the cluster and diagnose imbalanced read workloads or instances experiencing abnormally high read proposal rates.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 402, "dashboard": "PD-Cluster-Clinic", "title": "Store available", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the number of available stores in the TiKV cluster from PD's perspective. Essential for diagnosing cluster capacity issues, identifying unavailable stores that may impact data placement, and verifying store health before performing balance operations or troubleshooting replica scheduling problems.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 404, "dashboard": "PD-Cluster-Clinic", "title": "Store available ratio", "group": "Statistics - balance", "used_for": "Used for monitoring the ratio of available storage capacity to total capacity for each TiKV store. Critical for identifying stores approaching capacity limits and preventing out-of-space issues that could impact cluster availability and data balance.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_capacity\"}) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 469, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "", "used_for": "Used for monitoring the rate of event feed errors per minute in TiCDC's KV client, categorized by error type. Critical for identifying replication issues, connection problems, or data synchronization failures that could impact downstream data consistency.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 138, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiKV instance. Helps identify unbalanced write workloads across the cluster and diagnose instances experiencing abnormally high or low write proposal rates that could indicate performance bottlenecks or leader distribution issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 135, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile number of Raft proposals processed per ready batch in TiKV. Helps diagnose Raft proposal batching efficiency and identify potential bottlenecks in the consensus layer when proposals aren't being batched effectively, which can impact write throughput and latency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 132, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store io task reschedule", "group": "Raft IO", "used_for": "Used for monitoring the frequency of Raft IO task rescheduling in TiKV store threads, tracking both region rescheduling events and pending task rescheduling. Essential for diagnosing IO thread contention and identifying when the store thread is overloaded, causing tasks to be deferred or rescheduled rather than processed immediately.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 466, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD operator/m", "group": "", "used_for": "Used for monitoring the rate of PD scheduling operators created per minute, broken down by operator type. Helps identify scheduling activity patterns and diagnose issues related to region balancing, leader transfers, and other cluster rebalancing operations in TiDB.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 141, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the wait time before Raft store write tasks are executed in TiKV. Critical for diagnosing write path bottlenecks and identifying when the Raft store thread is overloaded, causing proposals to queue up before processing.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 405, "dashboard": "PD-Cluster-Clinic", "title": "Size amplification", "group": "Statistics - balance", "used_for": "Used for monitoring the storage amplification ratio in TiKV stores by comparing region size to available disk space. Helps identify stores with inefficient space utilization, excessive data fragmentation, or compaction issues that may lead to premature disk space exhaustion.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_used\"}) by (address, store) * 2^20" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 406, "dashboard": "PD-Cluster-Clinic", "title": "Store Region score", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the region distribution score across TiKV stores to evaluate load balance. Helps diagnose uneven region distribution issues by showing each store's region score, the scheduler's influence on balancing operations, and tolerance thresholds—critical for identifying why PD may not be balancing regions effectively.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_score\"}", "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}", "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 473, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV min resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiKV's resolved timestamp and the cluster's current TSO (Timestamp Oracle) in TiCDC change data capture scenarios. Critical for identifying data replication delays and diagnosing CDC pipeline bottlenecks that could impact downstream data synchronization freshness.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 408, "dashboard": "PD-Cluster-Clinic", "title": "Store Region size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of regions stored on each TiKV store in the cluster. Essential for identifying storage imbalance issues and determining if region distribution needs rebalancing across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 486, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write rows/s", "group": "", "used_for": "Used for monitoring the rate of rows written per second by TiCDC sinks to downstream systems. Essential for tracking replication throughput and identifying bottlenecks in data synchronization performance when changefeeds are running slowly or experiencing backpressure.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 485, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC sink operations, including both transaction flush duration and DDL execution time when writing to downstream systems. Critical for diagnosing replication lag issues and identifying performance bottlenecks in the data synchronization pipeline between TiDB and target databases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace,changefeed))", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace, changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 140, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Propose wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th and 99.99th percentile wait time before Raft proposals are processed on each TiKV instance. Critical for diagnosing write latency issues caused by Raft proposal queuing, helping identify instances experiencing backpressure or slow proposal processing that impacts write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 142, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th and 99.99th percentile wait time before Raft store write tasks are executed on each TiKV instance. Critical for diagnosing write bottlenecks caused by store thread saturation or scheduling delays in the Raft propose pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 407, "dashboard": "PD-Cluster-Clinic", "title": "Store leader score", "group": "Statistics - balance", "used_for": "Used for monitoring the leader score of each TiKV store, which represents the store's capacity to host region leaders based on its load and available resources. Essential for diagnosing leader distribution imbalances and understanding why PD schedules leaders to specific stores during load balancing operations.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_score\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 401, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scan activity patterns and diagnose potential bottlenecks in TiCDC's data replication process when capturing incremental changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 479, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiDB query duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile query execution latency in TiDB, excluding internal queries and simple commands. Critical for identifying tail latency issues and detecting slow query outliers that may impact user experience, particularly useful when diagnosing performance degradation in TiCDC replication scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[2m])) by (le, sql_type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 409, "dashboard": "PD-Cluster-Clinic", "title": "Store leader size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of Raft leader regions on each TiKV store. Essential for diagnosing leader distribution imbalances that can cause hotspots and uneven load across the cluster.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 411, "dashboard": "PD-Cluster-Clinic", "title": "Store witness count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of witness replicas on each TiKV store in the cluster. Helps verify witness replica distribution balance and diagnose placement issues when using witness replicas for disaster recovery configurations.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"witness_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 407, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV leader change", "group": "", "used_for": "Used for monitoring the rate of Raft leader region changes in TiKV instances over 2-minute intervals. Helps identify cluster instability, frequent leader transfers, or potential network issues that cause excessive leader switching, which can impact write performance and cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 410, "dashboard": "PD-Cluster-Clinic", "title": "Store Region count", "group": "Statistics - balance", "used_for": "Used for monitoring the distribution of regions across different TiKV stores in the cluster. Essential for identifying region imbalance issues that can lead to hotspots and uneven load distribution, helping diagnose performance bottlenecks caused by poor data placement.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 143, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft log entries spend waiting in the apply queue before being applied to the state machine in TiKV. Critical for diagnosing apply thread bottlenecks and identifying when the Raft apply process is falling behind, which can cause replication lag and write performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 467, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan region time/m", "group": "", "used_for": "Used for monitoring the total time TiKV spends scanning regions per minute for CDC (Change Data Capture) operations. Helps diagnose CDC performance issues and identify TiKV instances experiencing high region scan overhead during data replication.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 145, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write handle msg duration", "group": "Raft Propose", "used_for": "Used for monitoring the time TiKV's store thread spends handling write messages in the Raft consensus process. Critical for diagnosing Raft proposal bottlenecks and identifying when the store thread becomes a performance bottleneck for write operations.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 146, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write trigger size", "group": "Raft Propose", "used_for": "Used for monitoring the distribution of write batch sizes that trigger store-level writes in TiKV's Raft store. Helps diagnose write batching efficiency and identify whether write operations are being triggered at optimal sizes, which impacts I/O performance and write amplification.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 412, "dashboard": "PD-Cluster-Clinic", "title": "Store leader count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of Region leaders distributed across each TiKV store in the cluster. Essential for diagnosing leader distribution imbalances that can cause hotspots and uneven load distribution, helping identify when PD's leader scheduling needs adjustment.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 502, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write regions across TiKV stores from a peer perspective. Helps identify unbalanced hot region distribution and potential write hotspots that could cause performance bottlenecks or uneven load across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_peer\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 501, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's leader distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write region leaders across TiKV stores. Helps identify unbalanced write hotspots and leader placement issues that can cause performance bottlenecks or overload specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 144, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft log entries waiting in the apply queue before being applied to the state machine on each TiKV instance. Critical for diagnosing Raft apply bottlenecks and identifying instances experiencing delays in committing writes to storage, which directly impacts write latency and transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 470, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV admin apply/s", "group": "", "used_for": "Used for monitoring the rate of successful Raft admin commands applied per second in TiKV, broken down by command type (excluding compaction). Essential for tracking cluster administrative operations like region splits, merges, and configuration changes, helping diagnose issues with cluster topology adjustments and metadata operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 147, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft propose speed", "group": "Raft Propose", "used_for": "Used for monitoring the rate at which Raft proposals are being generated on each TiKV instance, measured in bytes per second. Helps identify instances with high write throughput and diagnose potential bottlenecks in the Raft consensus layer when proposal speeds are abnormally high or imbalanced across nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 152, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Replica read lock checking duration", "group": "Raft Process", "used_for": "Used for monitoring the time spent checking locks during replica read operations in TiKV. Essential for diagnosing latency issues with follower reads and identifying lock contention that may be blocking read requests from replica nodes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 400, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV unresolved region count", "group": "", "used_for": "Used for monitoring the number of TiKV regions that CDC has not yet resolved for change data capture. Critical for diagnosing CDC replication lag and identifying regions where change events are not being processed, which can indicate network issues, TiKV overload, or CDC synchronization problems.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 148, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile latency of RocksDB performance context operations during Raft store and apply processes in TiKV. Critical for diagnosing storage engine bottlenecks and understanding the time spent in RocksDB internal operations when processing Raft proposals.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 433, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader region count percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile of region count per check leader request in TiKV instances. Critical for identifying outlier scenarios where TiCDC or other components query an unusually large number of regions in a single check leader operation, which can indicate region distribution imbalances or inefficient scanning patterns.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 471, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resovled ts/s", "group": "", "used_for": "Used for monitoring the rate at which TiKV advances resolved timestamps per second for CDC (Change Data Capture). Critical for diagnosing CDC replication lag and identifying TiKV instances that are slow to advance their resolved timestamps, which can bottleneck downstream data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) " ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 503, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate bytes", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring write throughput (bytes per second) across TiKV stores to identify hot write spots and unbalanced write distribution. Essential for diagnosing write hotspot issues, evaluating load balancing effectiveness, and determining if specific stores are experiencing disproportionate write pressure that could impact cluster performance.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 427, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resolved ts fail/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of resolved timestamp advancement failures in TiKV instances, broken down by failure reason. Critical for diagnosing TiCDC replication lag issues and identifying which TiKV nodes are blocking CDC's ability to track data changes in real-time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 150, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft Process", "used_for": "Used for monitoring the rate at which Raft ready messages are processed in TiKV's Raftstore. Essential for diagnosing Raft consensus performance issues, identifying bottlenecks in log replication, and detecting when TiKV instances struggle to keep up with Raft message processing workload.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 504, "dashboard": "PD-Cluster-Clinic", "title": "Total written keys on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of keys written per second on hot leader Regions in the cluster. Helps identify write hotspots and unbalanced write loads across stores, which can indicate the need for Region splitting or load rebalancing to prevent performance bottlenecks.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_keys_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 399, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table rebalance", "group": "", "used_for": "Used for monitoring table redistribution activity across TiCDC instances during rebalancing operations. Helps identify when tables are being moved between capture nodes, which is essential for diagnosing changefeed stability issues and understanding load distribution changes in the TiCDC cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 508, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache write entry numbers", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of hot write regions cached in PD's hotspot detection system. Essential for diagnosing write hotspot issues and understanding how many regions are experiencing concentrated write traffic that may require load balancing.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 472, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader duration percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile latency of TiKV's leader checking operations in TiCDC's resolved timestamp mechanism. Critical for diagnosing tail latency issues in change data capture replication, particularly when TiCDC experiences delays in tracking data changes or leader verification becomes a bottleneck.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 151, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max duration of raft store events", "group": "Raft Process", "used_for": "Used for monitoring the maximum (p999999) latency of Raft store events in TiKV, including event processing and broadcast operations. Critical for diagnosing extreme tail latency issues in Raft consensus operations that could indicate store thread contention, slow disk I/O, or network problems affecting cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 999999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 155, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply fsm schedule wait duration", "group": "Raft Process", "used_for": "Used for monitoring the time Raft apply finite state machines (FSMs) spend waiting to be scheduled in TiKV's batch system. Critical for diagnosing apply thread pool saturation and identifying bottlenecks in Raft log application that can cause replication lag and write latency issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 505, "dashboard": "PD-Cluster-Clinic", "title": "Total written bytes on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total write throughput (in bytes) on hot leader Regions across stores. Essential for identifying write hotspots, detecting unbalanced write loads, and diagnosing performance bottlenecks caused by concentrated write traffic on specific leader Regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_bytes_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 154, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store fsm schedule wait duration", "group": "Raft Process", "used_for": "Used for monitoring the time store finite state machines (FSMs) spend waiting to be scheduled in TiKV's batch system. Critical for diagnosing Raft store thread contention and identifying bottlenecks in the store FSM scheduling pipeline that can impact write performance and replication latency.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10038, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV CDC Incremental Scan Long Duration Region Count", "group": "", "used_for": "Used for monitoring the number of TiKV regions experiencing prolonged incremental scan durations during CDC (Change Data Capture) operations. Critical for identifying regions with slow scan performance that could cause CDC replication lag or bottlenecks in change data synchronization.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "tikv_cdc_scan_long_duration_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 157, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply fsm poll duration", "group": "Raft Process", "used_for": "Used for monitoring the time distribution of apply finite state machine (FSM) polling operations in TiKV's Raft implementation. Critical for diagnosing Raft apply thread performance issues and identifying bottlenecks in the log application process that could impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 727, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Events Flush Lag", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the replication lag between TiCDC receiving change events and successfully flushing them to downstream sinks at different percentiles (p80, p95, p100). Critical for diagnosing data replication delays and identifying performance bottlenecks in the CDC pipeline that could impact downstream data freshness.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.8, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 19, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring the uptime duration of TiCDC, TiKV, and PD components since their last restart. Essential for tracking service availability, identifying unexpected restarts, and verifying cluster stability after maintenance or deployments.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tikv\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 506, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate keys", "group": "Statistics - hot write", "used_for": "Used for monitoring the write throughput in keys per second across TiKV stores, tracking both individual store write rates and region-level write rates. Essential for identifying hot write spots, diagnosing unbalanced write distribution, and detecting stores experiencing abnormally high write key traffic that may require load rebalancing.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 27, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Open FD count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiCDC processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiCDC to fail or become unstable.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 153, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fsm reschedule ops", "group": "Raft Process", "used_for": "Used for monitoring the rate of Finite State Machine (FSM) reschedule operations in TiKV's batch system, broken down by type. Helps diagnose Raft processing efficiency issues and identify potential bottlenecks when FSMs need to be rescheduled due to workload imbalances or resource contention.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_batch_system_fsm_reschedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store fsm poll round", "group": "Raft Process", "used_for": "Used for monitoring the distribution of polling rounds in TiKV's store finite state machine (FSM) batch system. Helps diagnose store thread efficiency and identify potential bottlenecks when FSMs require excessive polling rounds to process Raft store operations.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 159, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply fsm poll round", "group": "Raft Process", "used_for": "Used for monitoring the distribution of polling rounds in TiKV's apply finite state machine (FSM) batch system. Helps diagnose apply thread efficiency and identify potential bottlenecks when FSMs require excessive polling rounds to process Raft log entries.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 507, "dashboard": "PD-Cluster-Clinic", "title": "Store write query", "group": "Statistics - hot write", "used_for": "Used for monitoring the write query rate at the store level and identifying hot write regions in the cluster. Essential for diagnosing write hotspot issues, unbalanced load distribution across TiKV stores, and planning capacity or data rebalancing strategies.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 156, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store fsm poll duration", "group": "Raft Process", "used_for": "Used for monitoring the time distribution of store finite state machine (FSM) polling operations in TiKV's batch system. Critical for diagnosing Raft store thread performance issues and identifying bottlenecks in region metadata management and Raft message processing.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 160, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store fsm count per poll", "group": "Raft Process", "used_for": "Used for monitoring the distribution of finite state machines (FSMs) processed per poll in TiKV's store thread pool. Helps diagnose store thread efficiency and identify potential bottlenecks when FSM processing becomes unbalanced or excessively batched.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 509, "dashboard": "PD-Cluster-Clinic", "title": "Selector write events", "group": "Statistics - hot write", "used_for": "Used for monitoring the rate of hot write region detection events in PD's scheduler. Helps identify when and how frequently PD detects write hotspots across stores, which is essential for diagnosing unbalanced write load distribution and evaluating hotspot scheduling effectiveness.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 512, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot write", "used_for": "Used for monitoring the number of pending items in PD's hot write region detection queue. Helps diagnose delays in hot region identification and potential bottlenecks in PD's hot region scheduling mechanism when the queue accumulates items.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 24, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CPU usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiCDC server processes and their configured CPU limits (GOMAXPROCS). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiCDC nodes need vertical scaling or if CPU quotas are appropriately configured.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}[2m])", "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 161, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply fsm count per poll", "group": "Raft Process", "used_for": "Used for monitoring the distribution of apply finite state machines (FSMs) processed per poll in TiKV's Raft apply worker threads. Helps diagnose Raft apply throughput and identify whether the apply worker is processing FSMs efficiently or experiencing batching issues that could impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 511, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot write", "used_for": "Used for tracking PD's hotspot scheduling decisions for write-heavy regions, showing the direction and frequency of peer/leader movements across stores. Essential for diagnosing whether PD is actively balancing write hotspots and identifying stores that are frequently sources or destinations of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 510, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot write", "used_for": "Used for monitoring the direction and frequency of hot write region leader transfers between TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing write-intensive regions and identify stores that are frequently transferring leadership in or out due to write hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 162, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft message sizes exchanged between TiKV peers. Helps diagnose network bandwidth issues and identify abnormally large Raft messages that could impact replication performance and cluster communication efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 113, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership", "group": "Server", "used_for": "Used for monitoring the rate of ownership changes in TiCDC, tracking how frequently the owner role transfers between TiCDC nodes. Essential for diagnosing cluster stability issues, as frequent ownership changes may indicate node failures, network problems, or configuration issues affecting TiCDC's high availability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[240s])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiCDC server memory consumption, tracking both resident memory (RSS) and Go heap allocation. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC resource allocation during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 163, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft apply message batch sizes in TiKV. Helps diagnose Raft log application efficiency and identify whether messages are being batched optimally, which impacts write throughput and replication performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Goroutine count", "group": "Server", "used_for": "Used for monitoring the number of goroutines and OS threads in TiCDC instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate performance degradation or stability issues in the change data capture service.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 293, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD leader history", "group": "Server", "used_for": "**Used For:**\n\nUsed for tracking PD leader election history and identifying when leadership transfers occur in the cluster. Essential for diagnosing cluster stability issues, unexpected leader switches, and correlating PD leadership changes with performance anomalies or service disruptions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 602, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate bytes", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in bytes per second across TiKV stores, comparing actual store read rates with expected hot region read rates. Essential for identifying read hotspots, diagnosing unbalanced read traffic distribution, and validating whether PD's hot region scheduling is effectively balancing read-intensive workloads across stores.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 164, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Enable apply unpersisted log region count", "group": "Raft Process", "used_for": "**Used For:**\n\nUsed for monitoring the number of regions that have enabled the unpersisted apply log feature in TiKV's Raft implementation. Helps track the adoption of this optimization feature across regions and diagnose potential issues related to asynchronous log application and data persistence behavior.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 603, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate keys", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in terms of keys per second for each TiKV store, helping identify hot read spots and load imbalances across stores. Essential for diagnosing uneven read distribution and determining if specific stores are experiencing disproportionate read key traffic that could impact cluster performance.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 110, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership history", "group": "Server", "used_for": "Used for tracking TiCDC owner role changes and leadership transfers across instances. Helps diagnose cluster stability issues, frequent leader elections, or ownership handoff problems that could impact replication continuity.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[240s])) by (instance) > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture table count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being captured by each TiCDC instance for replication. Helps identify table distribution imbalances across TiCDC nodes and diagnose issues when changefeeds fail to capture expected tables.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 604, "dashboard": "PD-Cluster-Clinic", "title": "Store read query", "group": "Statistics - hot read", "used_for": "Used for monitoring read query rates on TiKV stores to identify hot read regions and uneven query distribution. Essential for diagnosing read hotspots, load balancing issues, and determining if specific stores are handling disproportionate read traffic that could impact cluster performance.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 607, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache read entry numbers", "group": "Statistics - hot read", "used_for": "Used for monitoring the total number of hot read regions cached in PD's hotspot detection system. Essential for understanding read hotspot distribution across stores and diagnosing unbalanced read traffic or hotspot scheduling issues.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 642, "dashboard": "Prometheus-TiCDC-Clinic", "title": "changefeed tables count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being replicated by each TiCDC changefeed. Helps identify table distribution across changefeeds and diagnose issues related to changefeed capacity, uneven table allocation, or replication scope problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 167, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages sent from each TiKV instance to other nodes in the cluster. Helps identify network communication patterns, detect instances with abnormally high message rates, and diagnose Raft replication issues or network bottlenecks between TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 605, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot read", "used_for": "Used for monitoring the direction and frequency of hot read region leader transfers across TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing read-heavy regions and identify stores that are frequently transferring leaders in or out to resolve read hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 513, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays in changefeeds, helping diagnose slow replication issues and ensuring downstream systems receive timely data updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 168, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft message flushes per TiKV instance, categorized by flush reason. Helps diagnose Raft communication efficiency and identify instances with abnormal message batching behavior that could impact cluster consensus performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 170, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages sent between TiKV nodes, broken down by message type and acceptance status. Essential for diagnosing Raft communication issues, network problems, and understanding cluster consensus behavior during replication or leader election scenarios.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 169, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages received by each TiKV instance. Helps identify communication imbalances, network issues, or instances under heavy Raft replication load that may indicate uneven data distribution or cluster health problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with real-time changes, helping diagnose replication bottlenecks or downstream processing issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 606, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot read", "used_for": "Used for tracking the direction and frequency of hot region movements (peer transfers and leader transfers) across TiKV stores during read hotspot balancing. Essential for diagnosing whether PD's hotspot scheduler is actively redistributing read-heavy regions and identifying which stores are sources or destinations of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 601, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot read", "used_for": "Used for monitoring the distribution of hot read regions across TiKV stores, showing both peer and leader roles. Helps identify load imbalance issues where hot regions are concentrated on specific stores, enabling targeted rebalancing decisions to optimize read performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_peer\"}", "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 403, "dashboard": "PD-Cluster-Clinic", "title": "Store used", "group": "Statistics - balance", "used_for": "Used for monitoring the actual disk space consumed by each TiKV store in the cluster. Essential for tracking storage utilization trends, identifying stores approaching capacity limits, and diagnosing uneven data distribution issues that may require rebalancing.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_used\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 608, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot read", "used_for": "Used for monitoring the number of pending items in PD's hot read region detection queue. Helps diagnose delays in hot region identification and potential bottlenecks in PD's hot region scheduling mechanism when the queue accumulates items.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 86, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint", "group": "Changefeed", "used_for": "Used for monitoring TiCDC changefeed replication progress by comparing the checkpoint timestamp against the current PD cluster TSO and barrier timestamp. Essential for identifying replication lag, detecting stuck changefeeds, and diagnosing data synchronization delays in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_owner_barrier_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 172, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages that are dropped in TiKV, categorized by message type. Critical for diagnosing Raft communication issues, network problems, or overloaded nodes that prevent proper consensus protocol operation and can lead to region unavailability or replication lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 701, "dashboard": "PD-Cluster-Clinic", "title": "Scheduler is running", "group": "Scheduler", "used_for": "Used for monitoring which schedulers are currently active and allowed to run in PD. Essential for verifying scheduler configuration changes and troubleshooting why certain data rebalancing or region scheduling operations may not be occurring as expected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"allow\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10037, "dashboard": "Prometheus-TiCDC-Clinic", "title": "GC Time", "group": "Changefeed", "used_for": "Used for monitoring TiCDC's garbage collection (GC) safe points, tracking both the minimum service GC safe point and CDC's own GC safe point. Critical for diagnosing data retention issues and ensuring TiCDC can access historical data needed for change data capture without being blocked by TiDB's GC mechanism.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(ticdc_gc_min_service_gc_safepoint{})", "max(ticdc_gc_cdc_gc_safepoint{})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 512, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts", "group": "Changefeed", "used_for": "Used for monitoring the replication progress of TiCDC changefeeds by comparing the resolved timestamp (the point up to which data has been replicated) against the current cluster TSO. Essential for identifying replication lag, detecting slow tables that are holding back the changefeed, and ensuring data synchronization is keeping pace with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 603, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Difference of resolvedTs and checkpointTs", "group": "Changefeed", "used_for": "Used for monitoring the replication lag in TiCDC by measuring the time difference between resolvedTs (latest transaction timestamp processed) and checkpointTs (latest timestamp successfully replicated downstream). Critical for identifying replication delays and ensuring data consistency between upstream TiDB and downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "(max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed) - max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)) / 1000 > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 609, "dashboard": "PD-Cluster-Clinic", "title": "Selector read events", "group": "Statistics - hot read", "used_for": "Used for monitoring the rate of hot read region detection events in PD's scheduler. Helps identify when and how frequently PD is detecting read hotspots across stores, which is essential for diagnosing read traffic imbalances and evaluating the effectiveness of hot region scheduling.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 703, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region movement", "group": "Scheduler", "used_for": "Used for monitoring the movement of regions during balance operations in PD's region scheduler. Helps diagnose load balancing effectiveness by showing which stores are sources (losing regions) and targets (gaining regions), essential for identifying uneven data distribution and scheduler behavior issues.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 163, "dashboard": "Prometheus-TiCDC-Clinic", "title": "The status of changefeeds", "group": "Changefeed", "used_for": "Used for monitoring the operational status of TiCDC changefeeds across the cluster. Essential for quickly identifying which changefeeds are running, stopped, or experiencing issues, enabling rapid troubleshooting of data replication problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 171, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Vote", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft vote messages sent by each TiKV instance. Essential for diagnosing leader election issues, cluster instability, or network partitions that trigger frequent voting rounds in the Raft consensus protocol.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 165, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply ahead of persistence raft log count", "group": "Raft Process", "used_for": "Used for monitoring the number of Raft log entries that have been applied to the state machine but not yet persisted to disk, tracking the p99 and maximum values. Critical for diagnosing potential data consistency risks and identifying instances where the apply process is significantly ahead of persistence, which could lead to data loss during crashes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 702, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader movement", "group": "Scheduler", "used_for": "Used for monitoring the balance leader scheduler's region leader movement between TiKV stores. Helps diagnose leader distribution issues by showing which stores are losing leaders (source) and which are gaining leaders (target), essential for troubleshooting uneven load distribution and scheduler effectiveness.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 704, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's balance leader scheduler, which redistributes region leaders across TiKV nodes to balance load. Essential for diagnosing leader distribution imbalances and verifying that the scheduler is actively working to equalize leadership across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 174, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft administrative operations in TiKV, specifically configuration changes and leader transfers. Essential for diagnosing cluster rebalancing activities, troubleshooting region leadership issues, and understanding the frequency of topology changes during scaling or maintenance operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 175, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft administrative commands (excluding compaction) being applied in TiKV, broken down by command type. Essential for diagnosing region management operations like splits, merges, and configuration changes that affect cluster topology and performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 102, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD etcd requests/s", "group": "Changefeed", "used_for": "Used for monitoring the rate of etcd requests per second made by TiCDC instances to PD's etcd. Helps identify excessive etcd access patterns, potential performance bottlenecks in metadata operations, and diagnose changefeed coordination issues.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 177, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft Admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiKV's Raft store. Critical for diagnosing extreme tail latency in region split operations, which can indicate issues with region management, metadata operations, or resource contention affecting cluster scalability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 705, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region scheduler", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the activity rate of PD's balance-region scheduler, which redistributes regions across TiKV stores to achieve balanced load. Essential for diagnosing uneven data distribution issues and verifying that region rebalancing is actively working to optimize cluster resource utilization.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 82, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Exit error count/m", "group": "Changefeed", "used_for": "Used for monitoring the rate of TiCDC processor exits due to errors per minute across changefeeds. Critical for detecting changefeed failures and replication interruptions that require immediate attention.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 514, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schema Storage GC progress", "group": "Changefeed", "used_for": "Used for monitoring the schema storage garbage collection progress in TiCDC changefeeds by comparing the GC timestamp against the cluster's current TSO (Timestamp Oracle). Essential for identifying schema storage cleanup delays that could lead to memory pressure or stale schema retention issues in CDC replication pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_processor_schema_storage_gc_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 706, "dashboard": "PD-Cluster-Clinic", "title": "Balance Hot Region scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's hot region balance scheduler, which automatically redistributes regions with high I/O load across TiKV nodes. Essential for diagnosing uneven I/O distribution issues and verifying whether the scheduler is actively balancing hot spots in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 218, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when event processing slows down or stalls.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 707, "dashboard": "PD-Cluster-Clinic", "title": "Split Bucket scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's split-bucket scheduler, which automatically splits hot Regions into smaller buckets to improve load balancing. Essential for diagnosing uneven data distribution issues and verifying that the scheduler is actively working to distribute hotspot workloads across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"split-bucket-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 180, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer in Flashback State", "group": "Raft Admin", "used_for": "Used for monitoring the number of Raft peers currently in flashback state on each TiKV instance. Essential for tracking the progress and scope of flashback operations, helping diagnose issues when restoring a cluster to a previous point in time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 712, "dashboard": "PD-Cluster-Clinic", "title": "Region merge checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of region merge operations checked by PD's merge checker. Essential for diagnosing region merge performance issues, identifying merge bottlenecks, and understanding whether regions are being consolidated efficiently to reduce cluster overhead.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"merge_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 178, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split event", "group": "Raft Admin", "used_for": "Used for monitoring the frequency of load-based region split events in TiKV, categorized by split type. Helps diagnose whether the cluster is automatically splitting regions appropriately in response to load pressure, which is critical for identifying hotspot issues and evaluating load balancing effectiveness.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 597, "dashboard": "Prometheus-TiCDC-Clinic", "title": "changefeed tick duration percentile", "group": "Changefeed", "used_for": "Used for monitoring the 99th and 99.9th percentile latency of TiCDC changefeed tick operations, which represent the core processing cycle for capturing and replicating data changes. Essential for diagnosing replication lag issues and identifying performance bottlenecks in CDC data synchronization pipelines.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_owner_changefeed_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))", "histogram_quantile(0.99, sum(rate(ticdc_processor_processor_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 176, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check split", "group": "Raft Admin", "used_for": "Used for monitoring the rate of region split checks in TiKV's Raftstore, categorized by check result type (success, failure, etc.). Essential for diagnosing region split behavior and identifying issues with automatic region splitting when data growth causes regions to exceed size thresholds.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 709, "dashboard": "PD-Cluster-Clinic", "title": "Patrol Region time", "group": "Scheduler", "used_for": "Used for monitoring the maximum time taken by PD to patrol (scan) all regions in the cluster. Critical for diagnosing scheduling delays and detecting when region metadata inspection becomes slow, which can impact PD's ability to balance load and respond to cluster changes.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(max(pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})by(instance))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 229, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events", "group": "Dataflow", "used_for": "Used for monitoring the number of events collected and output by TiCDC's puller component across changefeeds. Essential for tracking data replication throughput and diagnosing data flow bottlenecks or stalls in the CDC pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 220, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component across changefeeds. Essential for diagnosing data replication throughput issues and identifying bottlenecks in the event processing pipeline when replication lag occurs.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, instance, changefeed, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 179, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split duration", "group": "Raft Admin", "used_for": "Used for monitoring the latency of load-based region split operations in TiKV, showing p80, p99, and average durations. Critical for diagnosing performance issues when TiKV automatically splits regions based on load pressure, helping identify bottlenecks in region rebalancing and hotspot mitigation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 711, "dashboard": "PD-Cluster-Clinic", "title": "Replica checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of replica checker events in PD's scheduler. Helps diagnose replica placement issues, such as missing replicas, misplaced replicas, or regions that need rebalancing to meet configured replica constraints.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"replica_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 228, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component per second. Essential for tracking data flow throughput and identifying bottlenecks in the change data capture pipeline that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 184, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write operations", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection write operations per TiKV instance. Helps diagnose Raft log cleanup efficiency and identify instances with abnormal GC activity that could impact storage space management or I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 710, "dashboard": "PD-Cluster-Clinic", "title": "Rule checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of rule checker events in PD's scheduler, broken down by rule name. Helps diagnose placement rule violations and scheduling issues related to replica placement policies, label constraints, and isolation requirements.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"rule_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 183, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC kv sync duration", "group": "Raft Log", "used_for": "Used for monitoring the 99.99th percentile latency of Raft log synchronization to RocksDB's key-value store in TiKV. Critical for diagnosing extreme tail latency in Raft log persistence, which can indicate disk I/O bottlenecks or storage performance degradation affecting consensus replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 708, "dashboard": "PD-Cluster-Clinic", "title": "Balance Direction", "group": "Scheduler", "used_for": "Used for monitoring the direction and rate of data balance operations in PD's scheduler, showing data movement between stores (source to target) by operation type. Essential for diagnosing load balancing behavior, identifying hotspot migration patterns, and verifying that data is being redistributed correctly across the cluster.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (type, source, target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 713, "dashboard": "PD-Cluster-Clinic", "title": "Filter source", "group": "Scheduler", "used_for": "Used for monitoring the rate at which PD's scheduler filters out source stores during region scheduling operations. Helps diagnose why certain stores are being excluded as source candidates for region transfers, which is critical for troubleshooting unbalanced load distribution and scheduling inefficiencies.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (source, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 182, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write duration", "group": "Raft Log", "used_for": "Used for monitoring the 99.99th percentile write latency during Raft log garbage collection operations in TiKV. Critical for diagnosing performance issues when TiKV cleans up old Raft logs, as high GC write duration can indicate disk I/O bottlenecks or storage layer problems affecting cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 801, "dashboard": "PD-Cluster-Clinic", "title": "scatter operator event", "group": "Scatter and Splitter", "used_for": "Used for monitoring the rate of scatter operator events in PD, tracking successful, failed, and skipped scatter operations per minute. Essential for diagnosing region distribution issues and understanding why scatter operations may be failing or being skipped during load balancing or table creation.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"skip\"}[2m])*60) by (event)", "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fail\"}[2m])*60", "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"success\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 714, "dashboard": "PD-Cluster-Clinic", "title": "Filter target", "group": "Scheduler", "used_for": "Used for monitoring why PD's scheduler filters out potential target stores when creating operator schedules. Helps diagnose scheduling failures by showing which filters (distinct-filter, rule-fit-filter, etc.) are rejecting target stores, essential for troubleshooting why regions aren't being scheduled to expected stores.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (target, type, scope)", "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"distinct-filter\"}[2m])*60) by (source, target, type, scope)", "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[2m])*60) by (source, target, type, scope)", "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[2m])*60) by (source, target, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 223, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of data change events being written to downstream tables by TiCDC's table sink component. Essential for tracking replication throughput and identifying bottlenecks in the data synchronization pipeline when events are not being delivered at expected rates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 219, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's mounter component per second. Essential for tracking data processing throughput in the change data capture pipeline and identifying bottlenecks in event transformation from raw KV changes to structured row changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 224, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of row change events output by TiCDC's mounter component, which parses TiKV change data into structured events. Essential for tracking data replication throughput and diagnosing pipeline bottlenecks when changefeed lag increases or replication stalls occur.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 221, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of data change events output by TiCDC's table sink to downstream systems. Essential for tracking data replication progress, verifying event delivery rates, and diagnosing replication pipeline bottlenecks or stalls in CDC workflows.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 188, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC failed", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection failures in TiKV instances. Critical for identifying storage cleanup issues that can lead to disk space exhaustion and degraded cluster performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 186, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log lag", "group": "Raft Log", "used_for": "Used for monitoring the lag between Raft log replication and application in TiKV instances. Critical for identifying replication delays that can indicate slow followers, network issues, or instances struggling to keep up with the Raft consensus process.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 187, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log gc skipped", "group": "Raft Log", "used_for": "Used for monitoring the rate at which Raft log garbage collection is skipped in TiKV instances, categorized by reason. Essential for diagnosing issues where log compaction is being delayed or blocked, which can lead to excessive disk usage and potential performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 665, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows", "group": "Dataflow", "used_for": "Used for monitoring the total number of rows flushed to the downstream sink by TiCDC changefeeds. Essential for tracking data replication throughput and identifying potential bottlenecks or stalls in the data synchronization pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 185, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC seek operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of seek operations performed during Raft log garbage collection in TiKV. Helps diagnose I/O overhead and performance issues related to Raft log cleanup, where high seek rates may indicate inefficient log compaction or excessive fragmentation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 664, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of rows being flushed to downstream sinks per second in TiCDC changefeeds. Essential for tracking data replication throughput and identifying bottlenecks in the sink write pipeline when replication lag occurs.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 719, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Quota", "group": "Sink - General", "used_for": "Used for monitoring memory quota allocation and redo event cache usage in TiCDC's sink manager. Critical for diagnosing memory pressure issues, preventing OOM errors, and ensuring the changefeed has sufficient memory resources to handle data replication workloads.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sinkmanager_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", component!=\"\", instance=~\"$ticdc_instance\"}", "ticdc_sinkmanager_redo_event_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 189, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log fetch ", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log entry fetch operations in TiKV, categorized by fetch type. Helps diagnose Raft replication issues, log synchronization bottlenecks, and identify abnormal log fetching patterns that may indicate follower lag or network problems between TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_entry_fetches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 612, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Count / Second", "group": "Sink - General", "used_for": "Used for monitoring the rate of rows being written to downstream sinks by TiCDC per second. Essential for tracking replication throughput and identifying bottlenecks in data synchronization performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 902, "dashboard": "PD-Cluster-Clinic", "title": "gRPC 99% Completed commands duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC command completion in PD servers. Critical for identifying slow gRPC operations and diagnosing communication bottlenecks between PD and other TiDB components (TiKV, TiDB servers).", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, grpc_method, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 721, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Event Cache Hit Ratio", "group": "Sink - General", "used_for": "Used for monitoring the cache hit ratio of redo log events in TiCDC's sink manager. Helps diagnose redo log performance issues by showing how effectively events are being served from cache versus requiring disk I/O, with low ratios indicating potential memory pressure or inefficient caching.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", type=\"hit\"}[2m])) /\nsum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 193, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration", "group": "Raft Engine", "used_for": "Used for monitoring the write latency of TiKV's Raft Engine at p99 and p9999 percentiles. Critical for diagnosing Raft log persistence performance issues and identifying write bottlenecks that could impact transaction commit latency and overall cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 802, "dashboard": "PD-Cluster-Clinic", "title": "scatter store selection", "group": "Scatter and Splitter", "used_for": "Used for monitoring the distribution of scattered regions across TiKV stores, showing both leader and follower replica placement rates. Helps diagnose load balancing issues during table scatter operations and identify stores that may be receiving disproportionate region assignments, which could indicate scheduling problems or hotspot formation.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"false\"}[2m])*60) by (store)", "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"true\"}[2m])*60) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 195, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration Breakdown (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency breakdown of Raft Engine write operations across three stages: preprocessing, leader writing, and applying. Essential for diagnosing write performance bottlenecks in TiKV's Raft Engine by identifying which specific stage (preprocess, leader, or apply) is causing high write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 192, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Operation", "group": "Raft Engine", "used_for": "Used for monitoring the rate of I/O operations in TiKV's Raft Engine, including write apply, entry reads, and message reads. Essential for understanding Raft log storage activity patterns and diagnosing potential I/O bottlenecks or performance degradation in the consensus layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 903, "dashboard": "PD-Cluster-Clinic", "title": "gRPC commands concurrency number", "group": "gRPC", "used_for": "Used for monitoring the maximum concurrent gRPC API requests being processed by PD servers. Helps identify potential bottlenecks when PD is overwhelmed by concurrent client requests, which can lead to increased latency or request queuing issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"grpc\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 901, "dashboard": "PD-Cluster-Clinic", "title": "gRPC Completed commands rate", "group": "gRPC", "used_for": "Used for monitoring the rate of completed gRPC commands handled by each PD instance, broken down by gRPC method. Essential for tracking PD's API throughput, identifying which operations are most frequently called, and detecting sudden changes in client request patterns or potential bottlenecks in PD's gRPC service layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1001, "dashboard": "PD-Cluster-Clinic", "title": "HTTP Completed commands rate", "group": "HTTP", "used_for": "Used for monitoring the rate of completed HTTP commands handled by PD instances. Helps identify HTTP API traffic patterns, detect unusual spikes in command processing, and diagnose potential bottlenecks in PD's HTTP service layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,service,ip)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 194, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow", "group": "Raft Engine", "used_for": "**Used For:**\n\nUsed for monitoring the data flow rates in TiKV's Raft Engine, including write throughput and background rewrite operations. Essential for diagnosing Raft log write performance issues, identifying excessive rewrite activity, and understanding storage I/O patterns that may impact consensus protocol efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1002, "dashboard": "PD-Cluster-Clinic", "title": "HTTP 99% Completed commands duration", "group": "HTTP", "used_for": "Used for monitoring the 99th percentile latency of HTTP API commands handled by PD servers. Critical for identifying slow API responses and diagnosing PD service performance issues that may impact cluster management operations like scheduling, metadata queries, and TSO allocation.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, service, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 613, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Executed DDL Count / Minute", "group": "Sink - General", "used_for": "Used for monitoring the rate of DDL (Data Definition Language) operations executed by TiCDC sink per minute. Helps track schema change replication activity and identify potential bottlenecks or failures in DDL synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_sink_ddl_exec_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 610, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Batch Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile and average number of rows batched together in TiCDC sink output operations. Critical for diagnosing sink throughput efficiency and identifying whether batch sizes are optimal or if changefeeds are experiencing batching issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sinkv2_batch_row_count_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1003, "dashboard": "PD-Cluster-Clinic", "title": "HTTP commands concurrency number", "group": "HTTP", "used_for": "Used for monitoring the maximum concurrent HTTP API requests being processed by PD servers. Helps identify potential API overload situations and diagnose performance bottlenecks when PD's HTTP endpoints experience high concurrent access.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"http\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 190, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log async fetch task duration", "group": "Raft Log", "used_for": "Used for monitoring the latency of asynchronous Raft log fetching tasks in TiKV, tracking both the 99.99th percentile duration and average task completion time. Critical for diagnosing extreme tail latency in Raft log replication and identifying bottlenecks in the raftlog-fetch-worker that could impact consensus performance and follower read efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 614, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink Error Count (1min)", "group": "Sink - General", "used_for": "Used for monitoring the rate of sink execution errors in TiCDC over a 1-minute window. Critical for identifying data replication failures to downstream systems and diagnosing changefeed health issues that could cause data loss or replication lag.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10040, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL Executing Duration", "group": "Sink - General", "used_for": "Used for monitoring the p99 execution latency of DDL operations in TiCDC's sink component when replicating schema changes to downstream databases. Critical for diagnosing slow DDL replication issues and identifying bottlenecks in schema synchronization that could delay data replication.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_ddl_exec_duration_sum[1m])) by (le, namespace, changefeed, instance) / sum(rate(ticdc_sink_ddl_exec_duration_count[1m])) by (le, namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 198, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "File Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of log files, swap files, and recycled files in TiKV's Raft Engine storage layer. Helps diagnose file accumulation issues, storage cleanup efficiency, and potential disk space problems related to Raft log management.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 196, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Written", "group": "Raft Engine", "used_for": "Used for monitoring the distribution of write sizes to TiKV's Raft Engine, tracking p99 and p9999 percentiles plus average bytes written per operation. Essential for diagnosing write amplification issues and understanding the typical size of Raft log entries being persisted to disk.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 611, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Large Row (size >= 2K) Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile size of large rows (≥2KB) being replicated through TiCDC's sink component. Critical for identifying oversized row replication issues that may cause memory pressure, slow down replication throughput, or trigger downstream system bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_large_row_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_large_row_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sink_large_row_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 197, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL Duration Breakdown (999%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency breakdown of Raft Engine's Write-Ahead Log (WAL) operations in TiKV, including leader write, log sync, and log allocation phases. Critical for diagnosing tail latency issues in Raft log persistence and identifying which specific WAL operation stage is causing write performance bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 201, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Compression Ratio", "group": "Raft Engine", "used_for": "Used for monitoring the compression efficiency of Raft Engine write operations in TiKV, showing how much data is reduced during writes. Essential for evaluating storage efficiency and identifying compression performance issues that could impact disk I/O and storage capacity utilization.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1102, "dashboard": "PD-Cluster-Clinic", "title": "Handle transactions rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd transaction operations handled by PD instances, broken down by result status (success/failure). Essential for diagnosing PD's etcd transaction throughput and identifying potential bottlenecks or failures in metadata operations that could impact cluster coordination.", "component": "PD", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1105, "dashboard": "PD-Cluster-Clinic", "title": "99% WAL fsync duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile fsync latency of etcd's Write-Ahead Log (WAL) in PD. Critical for diagnosing PD performance issues caused by slow disk I/O, as high WAL fsync duration directly impacts PD's ability to persist cluster metadata and coordinate distributed transactions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 199, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Other Durations (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency of miscellaneous Raft Engine operations in TiKV, including reading entries, reading messages, and purging old data. Essential for identifying tail latency issues in Raft log management operations that could impact consensus performance and replication efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1103, "dashboard": "PD-Cluster-Clinic", "title": "99% Handle transactions duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in PD. Critical for diagnosing slow consensus operations and identifying performance degradation in PD's underlying etcd storage, which can impact cluster metadata operations and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 200, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Entry Count", "group": "Raft Engine", "used_for": "Used for monitoring the average number of log entries stored in Raft Engine, categorized by entry type. Helps diagnose Raft log accumulation issues, identify potential log compaction problems, and understand the distribution of different entry types that could impact TiKV's consensus protocol performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 620, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Full Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction flush operations in TiCDC's sink component. Critical for diagnosing downstream write performance issues and identifying when transaction batches take unusually long to commit to the target database.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 634, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Input Rows / Second", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the rate of rows being processed by TiCDC's transaction sink workers per second. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when rows are being written to downstream systems.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1101, "dashboard": "PD-Cluster-Clinic", "title": "MVCC DB total size", "group": "etcd", "used_for": "Used for monitoring the total size and actual usage of PD's embedded etcd MVCC database. Critical for tracking metadata storage growth and identifying potential disk space issues or excessive historical version accumulation that could impact PD performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}", "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1104, "dashboard": "PD-Cluster-Clinic", "title": "99% Peer round trip time seconds", "group": "etcd", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile network round-trip time between etcd peers in the PD cluster. Critical for diagnosing network latency issues, etcd consensus delays, and identifying slow peer connections that could impact PD's metadata operations and cluster coordination performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, To, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 619, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the busy ratio of transaction sink workers in TiCDC, indicating how much time workers spend flushing transactions versus being idle. Essential for identifying sink bottlenecks and determining if worker capacity needs adjustment when ratios consistently approach 100%.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) /sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) *100" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1107, "dashboard": "PD-Cluster-Clinic", "title": "99% backend commit duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd backend commit operations in PD. Critical for diagnosing PD performance issues caused by slow disk writes, as high commit latency directly impacts cluster metadata operations and can indicate storage bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1108, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk backend commit rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd backend commit operations per minute in PD. Essential for identifying etcd write throughput bottlenecks and diagnosing performance degradation in PD's metadata storage layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 203, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation efficiency by tracking where Get operations are served from (memtable, block cache, or L0 SST files). Helps diagnose read performance issues by identifying cache hit rates and whether reads are efficiently served from memory versus requiring disk I/O.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1607, "dashboard": "PD-Cluster-Clinic", "title": "99% Endpoint health check latency", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd endpoint health checks in PD. Critical for detecting etcd communication issues and cluster metadata access problems that could impact PD's ability to coordinate the cluster.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_etcd_endpoint_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}[2m])) by (instance, endpoint, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 635, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backend Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of DML batch commits to the downstream database in TiCDC's transaction sink. Critical for diagnosing slow replication issues and identifying backend database performance bottlenecks that may cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1109, "dashboard": "PD-Cluster-Clinic", "title": "Raft term", "group": "etcd", "used_for": "Used for monitoring the Raft term number in PD's embedded etcd cluster. Helps identify leader elections and cluster stability issues—frequent term changes indicate leadership instability or network partitions affecting consensus.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"term\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1106, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk wal fsync rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd WAL (Write-Ahead Log) fsync operations per minute in PD. Essential for identifying write throughput patterns and diagnosing potential disk I/O bottlenecks that could impact PD's etcd performance and cluster metadata operations.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 205, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation rates in TiKV, including total seeks, successful seeks found, and next operations. Essential for diagnosing read performance issues and understanding storage engine scan patterns, as excessive seeks may indicate inefficient queries or suboptimal data layout.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1110, "dashboard": "PD-Cluster-Clinic", "title": "Endpoint health state", "group": "etcd", "used_for": "Used for monitoring the health status of etcd client connections in PD. Critical for detecting etcd connectivity issues that could impact PD's ability to store and retrieve cluster metadata, helping diagnose cluster coordination failures.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_client{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 207, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB write operation rates in TiKV, including successful writes (self/other), write timeouts, and WAL writes. Essential for diagnosing write throughput issues, identifying write bottlenecks, and detecting abnormal write timeout patterns that may indicate storage layer performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 206, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiKV's storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing read performance issues and identifying inefficient range scans or index lookups that may indicate fragmentation or suboptimal query patterns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 653, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC workers sending messages to message queue (MQ) sinks like Kafka. Critical for diagnosing tail latency issues in data replication to downstream MQ systems and identifying potential bottlenecks in the CDC pipeline's message delivery performance.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 208, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write operation latency in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Essential for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or storage layer issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 204, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of RocksDB point lookup (Get) operations in TiKV, showing maximum, P99, and P95 durations. Essential for diagnosing read performance issues and identifying when individual key lookups are experiencing high latency in the storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 628, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Outgoing Bytes / s", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the outgoing data throughput rate (bytes per second) from TiCDC to Kafka message queues. Essential for tracking replication bandwidth usage, identifying network bottlenecks, and ensuring data is being successfully transmitted to downstream Kafka clusters at expected rates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 618, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Conflict Detect Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction conflict detection in TiCDC's transaction sink, along with average detection time and queue duration. Critical for diagnosing replication delays caused by transaction conflicts when TiCDC writes to downstream databases, helping identify whether slowdowns stem from conflict resolution overhead or queue backlog.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 209, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations in TiKV's RocksDB storage engine. Essential for diagnosing write durability overhead and identifying potential I/O bottlenecks caused by frequent fsync operations that ensure data persistence.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_wal_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 631, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Records Per Request", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of records batched per Kafka produce request in TiCDC's MQ sink. Helps evaluate batching efficiency and diagnose throughput issues—low values may indicate poor batching performance, while high values suggest optimal message aggregation.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 627, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Inflight Requests / s", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of in-flight (pending) requests per second to Kafka brokers from TiCDC's MQ sink. Critical for identifying backpressure issues, network bottlenecks, or when TiCDC is waiting for Kafka acknowledgments, which can indicate replication lag or Kafka cluster performance problems.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 211, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiKV, including flush, compaction, and other storage engine events. Essential for diagnosing write amplification issues, storage performance bottlenecks, and understanding background I/O activity that may impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_event_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1111, "dashboard": "PD-Cluster-Clinic", "title": "Raft applied index", "group": "etcd", "used_for": "Used for monitoring the Raft applied index in PD's embedded etcd cluster, which indicates the latest log entry that has been applied to the state machine. Essential for diagnosing etcd replication lag, consensus issues, or when troubleshooting PD cluster synchronization problems.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"appliedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 630, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Latency", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the latency of Kafka producer requests in TiCDC's message queue sink. Critical for diagnosing replication delays and identifying performance bottlenecks when TiCDC writes change data to Kafka topics.", "component": "TICDC", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 626, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Producer Compression Ratio", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the compression ratio achieved by Kafka producers in TiCDC's message queue sink. Helps evaluate data compression efficiency and diagnose issues with excessive network bandwidth usage or suboptimal compression settings when replicating changes to Kafka.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 212, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the Write-Ahead Log (WAL) sync latency in TiKV's RocksDB storage engine, tracking max, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues, as WAL sync directly impacts write transaction durability and throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 629, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Rate / s", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of requests sent to Kafka brokers per second by TiCDC's MQ sink. Essential for tracking data replication throughput to Kafka and diagnosing bottlenecks or performance degradation in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 210, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the time spent writing to RocksDB's Write-Ahead Log (WAL) in TiKV, showing max, P99, and P95 latencies. Critical for diagnosing write performance bottlenecks and disk I/O issues, as WAL write latency directly impacts transaction commit performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 709, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Encoder group input channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the input channel buffer size of TiCDC's encoder group when using message queue (MQ) sinks. Helps identify backpressure issues and bottlenecks in the encoding pipeline, where a growing channel size indicates the encoder cannot keep up with incoming change events.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 712, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Size Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile batch size of messages sent by TiCDC's MQ sink workers to message queues like Kafka. Critical for diagnosing message batching efficiency and identifying potential throughput bottlenecks when replicating data to downstream MQ systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 710, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Encoder group output channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the output channel buffer size of TiCDC's encoder group when replicating data to message queue (MQ) sinks. Helps identify backpressure issues and bottlenecks in the encoding pipeline, where a growing channel size indicates the encoder is producing data faster than the MQ sink can consume it.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1201, "dashboard": "PD-Cluster-Clinic", "title": "PD server TSO handle duration", "group": "TiDB", "used_for": "Used for monitoring the latency of PD server handling TSO (Timestamp Oracle) requests at p90, p99, and p999 percentiles. Critical for diagnosing transaction performance issues, as TSO allocation delays directly impact TiDB's ability to assign timestamps for distributed transactions and can cause cluster-wide latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1203, "dashboard": "PD-Cluster-Clinic", "title": "Handle requests count", "group": "TiDB", "used_for": "Used for monitoring the rate of TSO (Timestamp Oracle) requests handled by PD servers. Essential for tracking timestamp allocation activity and diagnosing issues when TiDB experiences transaction coordination problems or TSO service bottlenecks.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,dc)", "sum(rate(tso_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 711, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of MQ worker batch processing in TiCDC's message queue sink. Critical for identifying tail latency issues when TiCDC replicates data to downstream message queue systems like Kafka, helping diagnose slow batch processing and potential replication delays.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 213, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction guard actions", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of compaction guard actions in TiKV's RocksDB storage engine across default and write column families. Helps diagnose write stalls and storage layer performance issues caused by excessive compaction activity or pending compactions that trigger flow control mechanisms.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_compaction_guard_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 216, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file reads in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing storage layer performance issues, especially when queries experience slow reads from disk or when investigating I/O bottlenecks in the RocksDB data files.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1202, "dashboard": "PD-Cluster-Clinic", "title": "PD client requests handle duration", "group": "TiDB", "used_for": "Used for monitoring the latency of TiDB's requests to PD (Placement Driver) at different percentiles (p90, p99, p999). Critical for diagnosing communication issues between TiDB and PD, especially when experiencing slow metadata operations, TSO allocation delays, or region routing problems.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 215, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction Job Size(files)", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of SST files involved in each RocksDB compaction operation, showing maximum, P99, and P95 values. Essential for diagnosing compaction efficiency and detecting write amplification issues—abnormally high file counts indicate excessive compaction overhead that can impact write performance and I/O utilization.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 741, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Count / s", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of claim check messages sent per second by TiCDC's MQ sink. Helps track when TiCDC uses claim check pattern (storing large messages externally) to handle oversized change data, indicating potential message size optimization needs.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1204, "dashboard": "PD-Cluster-Clinic", "title": "Tso Request batch size", "group": "TiDB", "used_for": "Used for monitoring the batch size of TSO (Timestamp Oracle) requests from TiDB to PD, showing p90, p99, and average values. Helps diagnose TSO request batching efficiency and identify potential performance issues when batch sizes are abnormally small (indicating poor batching) or large (indicating request queuing).", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [ 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "sum(rate(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(rate(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1301, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat region event QPS", "group": "Heartbeat", "used_for": "Used for monitoring the rate of region heartbeat events received by PD from TiKV stores. Essential for diagnosing region scheduling issues, detecting abnormal region state changes, and identifying stores with excessive region operations that may impact cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1610, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat handle latency overview", "group": "Heartbeat", "used_for": "**Used For:**\n\nUsed for monitoring the latency distribution (p99, p90, p80) of PD handling region heartbeat requests from TiKV stores. Critical for diagnosing PD performance bottlenecks when region metadata updates are slow, which can impact cluster scheduling decisions and overall stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "histogram_quantile(0.9, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) / sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 214, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the time taken by RocksDB compaction operations in TiKV, showing maximum, P99, and P95 latencies. Critical for diagnosing write amplification issues and identifying when background compaction processes are impacting overall storage performance or causing write stalls.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 217, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons triggering RocksDB compactions in TiKV storage engine, broken down by column family and compaction trigger type. Essential for diagnosing write amplification issues, understanding storage layer behavior, and identifying whether compactions are triggered by size thresholds, manual operations, or other factors affecting I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_compaction_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (cf, reason, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 218, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB block cache across different column families in TiKV instances. Essential for diagnosing memory pressure issues and optimizing cache allocation to improve read performance and prevent OOM errors.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n avg((\n tikv_engine_block_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, instance) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 731, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Message Count Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the distribution of message counts per batch in TiCDC's Kafka producer when using the kafka-go library. Helps diagnose batching efficiency and identify whether messages are being properly aggregated before sending to Kafka, which impacts throughput and network efficiency.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 740, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of claim check message sending operations in TiCDC's MQ sink. Critical for diagnosing tail latency issues when TiCDC uses claim check pattern to handle large messages in message queue systems, helping identify performance bottlenecks in external storage interactions.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_claim_check_send_message_duration_sum[1m])) by (namespace, changefeed, instance) / sum(rate(ticdc_sink_mq_claim_check_send_message_duration_count[1m])) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 219, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the hit rate of RocksDB memtable lookups in TiKV. Helps diagnose memory efficiency and identify whether data reads are successfully served from in-memory memtables before checking SST files, which is critical for optimizing read performance.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1303, "dashboard": "PD-Cluster-Clinic", "title": "TiKV side heartbeat statistics", "group": "Heartbeat", "used_for": "Used for monitoring heartbeat communication between TiKV nodes and PD (Placement Driver), tracking both heartbeat message rates by type and pending heartbeat counts. Essential for diagnosing cluster coordination issues, detecting network problems between TiKV and PD, and identifying when TiKV nodes are experiencing delays in reporting their status to the cluster coordinator.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 220, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read and write traffic flow through RocksDB's block cache in TiKV. Essential for analyzing cache utilization patterns, identifying cache pressure, and diagnosing performance issues related to block cache efficiency and memory usage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 739, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Retry Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of message send retries in TiCDC's Kafka sink when using the kafka-go client library. Essential for identifying Kafka connectivity issues, broker availability problems, or network instability that causes message delivery failures requiring retries.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_retry_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 737, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Error Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's Kafka-Go producer when replicating data to Kafka sinks. Critical for identifying replication failures, connection issues, or message delivery problems that could cause data loss or replication lag.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_err_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1304, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster health problems that may affect region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"ok\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 735, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Bytes Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the maximum batch size in bytes for Kafka messages produced by TiCDC's kafka-go client. Helps diagnose message batching efficiency and identify potential issues with Kafka producer configuration or network throughput when replicating data to Kafka sinks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_size{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1609, "dashboard": "PD-Cluster-Clinic", "title": "Concurrent Runner Pending Task", "group": "Heartbeat", "used_for": "Used for monitoring the queue depth and maximum waiting time of rate-limited tasks in PD's concurrent runner. Critical for diagnosing heartbeat processing delays and identifying when PD is overwhelmed by pending tasks, which can lead to cluster coordination issues.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_ratelimit_runner_pending_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "pd_ratelimit_runner_task_max_waiting_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 222, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the rate of key operations in TiKV's RocksDB storage engine, tracking keys read, keys written, and corrupt keys detected during compaction. Essential for diagnosing data flow patterns, identifying write-heavy workloads, and detecting potential data corruption issues in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_compaction_num_corrupt_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 733, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Messages Duration Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the duration percentiles of Kafka message batching operations in TiCDC's Kafka-Go producer implementation. Essential for diagnosing message batching latency issues and identifying performance bottlenecks when replicating data changes to Kafka sinks.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_duration{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10017, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Flush duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the time taken to flush data files to cloud storage in TiCDC's cloud storage sink. Critical for diagnosing slow data replication to cloud destinations and identifying potential bottlenecks in the file upload process.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 224, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of RocksDB storage engine in TiKV, tracking both direct bytes read and iterator bytes read. Essential for diagnosing read I/O performance issues, identifying read-heavy workloads, and correlating storage layer read activity with query performance problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 715, "dashboard": "Prometheus-TiCDC-Clinic", "title": "File Count", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the total number of files generated by TiCDC's cloud storage sink for each changefeed instance. Helps track file accumulation patterns and identify potential issues with file generation rate or storage cleanup processes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10015, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the latency of write operations to cloud storage in TiCDC's cloud storage sink. Critical for diagnosing performance bottlenecks when replicating data to cloud storage destinations like S3 or GCS, helping identify slow writes that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 221, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the hit rate of RocksDB's block cache in TiKV, including overall cache hits, data block hits, and filter block hits. Essential for diagnosing read performance issues and determining if the block cache is sized appropriately—low hit rates indicate frequent disk I/O and potential need for cache tuning.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) / ((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) + (sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) )))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 225, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys stored in each RocksDB column family within TiKV. Helps assess data distribution, storage capacity planning, and identify potential data skew issues across different column families.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_engine_estimate_num_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 227, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the amount of data read per read operation in TiKV's RocksDB storage engine, showing maximum, P99, and P95 values. Helps diagnose read amplification issues and identify inefficient queries that scan excessive data, which can indicate problems with index usage or data layout.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1307, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report active", "group": "Heartbeat", "used_for": "Used for monitoring the rate of active region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster coordination problems that could impact region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"bind\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1608, "dashboard": "PD-Cluster-Clinic", "title": "Concurrent Runner Failed Task", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed tasks in PD's concurrent runner, which handles rate-limited operations like heartbeat processing. Essential for diagnosing PD performance issues and identifying when the cluster is experiencing task execution failures that could lead to increased latency or coordination problems.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_ratelimit_runner_failed_tasks_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 714, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write Bytes/s", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the write throughput (bytes per second) of TiCDC's cloud storage sink. Essential for tracking data replication performance to cloud storage and identifying bandwidth bottlenecks or write slowdowns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 226, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the write throughput in TiKV's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written per second. Essential for diagnosing write performance bottlenecks, identifying I/O saturation issues, and understanding storage layer write patterns during high-load scenarios.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1306, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report error", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed region heartbeat reports from TiKV stores to PD. Critical for identifying communication issues, network problems, or store health degradation that prevents proper cluster metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"err\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1309, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful heartbeat region messages pushed from PD to TiKV stores. Essential for diagnosing communication issues between PD and TiKV nodes, particularly when investigating cluster coordination problems or store connectivity failures.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"ok\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 571, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying replication delays, ensuring data freshness in downstream systems, and diagnosing changefeed performance issues or bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 223, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiKV's RocksDB storage engine, including overall block additions, data block additions, and filter block additions. Essential for understanding cache population patterns and diagnosing issues related to cache thrashing or inefficient cache utilization that could impact read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1335, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat Performance Duration BreakDown (Accumulation)", "group": "Heartbeat", "used_for": "Used for analyzing the time breakdown of PD's region heartbeat processing pipeline, showing accumulated duration across different processing stages (handle duration) and lock acquisition wait times. Essential for diagnosing PD heartbeat latency bottlenecks and identifying which specific stage (e.g., lock contention, processing overhead) is causing performance degradation in cluster metadata management.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_core_region_heartbeat_breakdown_handle_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (name)", "sum(rate(pd_core_acquire_regions_lock_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 229, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the average number of bytes written per write operation in TiKV's RocksDB storage engine, showing maximum, P99, and P95 percentiles. Helps diagnose write amplification issues and identify whether write operations are efficiently batched or fragmented into small writes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1308, "dashboard": "PD-Cluster-Clinic", "title": "99% store heartbeat handle duration", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling store heartbeat messages from TiKV nodes. Critical for diagnosing PD performance bottlenecks and detecting when the cluster controller is slow to process store status updates, which can impact scheduling decisions and cluster health detection.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 228, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the I/O throughput of RocksDB compaction operations in TiKV, tracking bytes read during compaction, bytes written after compaction, and flush write bytes. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification that may impact overall TiKV write performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 589, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Scheduler", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring near real-time replication to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1305, "dashboard": "PD-Cluster-Clinic", "title": "Region schedule push", "group": "Heartbeat", "used_for": "Used for monitoring the rate at which PD pushes region scheduling commands to TiKV stores via heartbeat messages. Essential for diagnosing scheduling delays and identifying stores that may be slow to receive or process region scheduling operations, which can impact cluster rebalancing and data migration performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1312, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report State", "group": "Heartbeat", "used_for": "Used for monitoring the rate and status of bucket reports sent from TiKV stores to PD. Helps diagnose heartbeat communication issues and identify stores with abnormal bucket reporting behavior that could indicate network problems or store health issues.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, type,status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 230, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read amplification", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiKV's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for diagnosing inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O during read operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 231, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the total bytes of data waiting to be compacted in RocksDB column families. Critical for identifying compaction backlogs that can lead to read amplification, write stalls, and degraded I/O performance in TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1311, "dashboard": "PD-Cluster-Clinic", "title": "99% Bucket Report Interval", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile interval between bucket heartbeat reports from TiKV stores to PD. Critical for detecting delayed or irregular heartbeat communications that could indicate network issues, store overload, or potential cluster coordination problems.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10019, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the busy ratio of cloud storage sink workers in TiCDC, indicating how much time workers spend actively processing data versus idle. Essential for identifying worker saturation issues and determining if additional worker capacity is needed to handle replication workload efficiently.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])*100) by (namespace,changefeed,id,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1310, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report error", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed region heartbeat messages from PD to TiKV stores. Critical for diagnosing communication issues between PD and TiKV that can cause cluster instability, stale region information, or scheduling delays.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"err\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 546, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table replication state", "group": "Scheduler", "used_for": "Used for monitoring the current replication state of tables being synchronized by TiCDC changefeeds. Helps identify tables that are stuck, stopped, or experiencing replication issues, enabling quick diagnosis of data synchronization problems in CDC pipelines.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 232, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiKV instances. Helps identify potential memory leaks or snapshot accumulation issues that can impact performance and resource usage, as excessive snapshots may indicate problems with snapshot cleanup or long-running transactions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_engine_num_snapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1302, "dashboard": "PD-Cluster-Clinic", "title": "99% Region heartbeat handle latency by store", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling region heartbeats from each TiKV store. Critical for diagnosing PD performance bottlenecks when stores report slow heartbeat processing, which can indicate PD overload or network issues affecting cluster metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 549, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks accept", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduling tasks accepted by TiCDC changefeeds. Helps identify task scheduling bottlenecks and verify that the scheduler is properly accepting and distributing replication tasks across TiCDC instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1401, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of read traffic across regions in bytes per second. Helps identify hot regions with high read load and diagnose unbalanced read traffic that may cause performance bottlenecks in the cluster.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 233, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression ratio of data stored in RocksDB across different column families and levels in TiKV. Helps assess storage efficiency and identify compression issues that could lead to excessive disk space usage or suboptimal I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_engine_compression_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1402, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of write traffic across regions in the PD cluster by tracking bytes written per region over time. Helps identify write hotspots, unbalanced region load distribution, and potential scheduling issues that could impact write performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 548, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks", "group": "Scheduler", "used_for": "Used for monitoring the distribution and volume of scheduling tasks across different schedulers in TiCDC. Helps identify scheduling bottlenecks, uneven task distribution, and potential performance issues in the change data capture replication pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 235, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiKV's RocksDB storage engine. Critical for identifying long-running transactions or snapshots that may block MVCC garbage collection, leading to disk space bloat and performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_engine_oldest_snapshot_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1404, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Key", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the rate of region key writes processed by PD's scheduler. Helps diagnose region scheduling performance and identify bottlenecks when PD is slow to update region metadata or handle region key operations.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 236, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingestion picked level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of LSM tree levels where ingested SST files are placed in RocksDB. Helps diagnose ingestion efficiency and potential write amplification issues by showing whether files are being placed at optimal levels in the storage hierarchy.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_engine_ingestion_picked_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 588, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks running", "group": "Scheduler", "used_for": "Used for monitoring the number of actively running scheduler tasks in TiCDC changefeeds. Helps identify scheduling bottlenecks, task accumulation issues, and abnormal task execution patterns that could impact data replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 545, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table count", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the total number of tables being replicated by TiCDC changefeeds. Helps identify changefeed scale, verify table distribution across the scheduler, and diagnose issues when tables are unexpectedly added or removed from replication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1405, "dashboard": "PD-Cluster-Clinic", "title": "Store Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of heartbeat intervals between TiKV stores and PD. Helps diagnose cluster health issues, network problems, or overloaded stores when heartbeat intervals become irregular or exceed expected thresholds.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1403, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Key", "group": "Heartbeat distribution ", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of region key read operations in PD's scheduler. Helps diagnose region metadata access patterns and identify potential bottlenecks in PD's region information retrieval, which can impact scheduling decisions and cluster coordination performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 234, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in RocksDB's LSM-tree structure for each column family. Essential for diagnosing compaction issues, identifying level imbalances, and understanding storage amplification problems that can impact TiKV read/write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_engine_num_files_at_level\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 237, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file ingestion operations in TiKV's RocksDB storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing performance issues during bulk data imports, snapshot applications, or when TiKV ingests external SST files from backup/restore or Lightning operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_ingest_external_file_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) / sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) )", "sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 547, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture tables count", "group": "Scheduler", "used_for": "Used for monitoring the number of tables assigned to each TiCDC capture node across changefeeds. Helps identify table distribution imbalances in the scheduler and diagnose potential workload skew issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1406, "dashboard": "PD-Cluster-Clinic", "title": "Region Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of time intervals between consecutive region heartbeats sent to PD. Essential for detecting region heartbeat irregularities, network issues, or TiKV health problems that could indicate cluster instability or communication delays.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 543, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Scheduler", "used_for": "Used for identifying which table ID is experiencing the slowest replication performance in TiCDC's scheduler. Helps pinpoint specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 238, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for identifying the root causes of write stalls in TiKV's RocksDB storage engine. Essential for diagnosing write performance degradation by showing which specific conditions (memtable full, L0 file limits, pending compaction bytes, etc.) are triggering flow control and blocking writes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_write_stall_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1407, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of bucket report intervals from TiKV stores to PD. Helps diagnose heartbeat delays and communication issues between TiKV and PD, which can indicate network problems or overloaded stores affecting cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 544, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table replication state", "group": "Scheduler", "used_for": "Used for identifying tables with the slowest replication progress in TiCDC changefeeds. Critical for diagnosing replication lag issues and pinpointing which specific tables are causing bottlenecks in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 245, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the memory usage of Titan's blob cache across different column families in TiKV. Essential for diagnosing memory pressure issues and optimizing blob cache configuration when using Titan storage engine for large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "topk(20,(\n avg((\n tikv_engine_blob_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (cf, instance) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 240, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for tracking the frequency of RocksDB stall condition changes across different column families in TiKV. Essential for diagnosing write throttling issues and identifying when RocksDB enters or exits stall states due to memtable limits, L0 file accumulation, or pending compaction bytes exceeding thresholds.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_engine_stall_conditions_changed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring query latency distribution across TiDB instances at multiple percentiles (p80, p95, p99, p999). Essential for identifying performance degradation, tail latency issues, and understanding query response time characteristics under different load conditions.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1408, "dashboard": "PD-Cluster-Clinic", "title": "Hot Degree of Bucket", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of hot bucket degrees in PD's scheduler, indicating how many buckets are experiencing high access frequency. Helps identify hotspot regions and diagnose load balancing issues when certain buckets show consistently high heat levels.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 590, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table resolved ts", "group": "Scheduler", "used_for": "Used for monitoring the resolved timestamp of the slowest table in TiCDC changefeeds, compared against the cluster's current TSO. Critical for identifying replication lag bottlenecks caused by specific slow tables that are holding back the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 541, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table checkpoint", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for identifying the slowest table's replication checkpoint timestamp in TiCDC changefeeds compared to the current PD TSO. Critical for detecting replication lag bottlenecks caused by specific tables that are falling behind, helping pinpoint which tables are slowing down the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 241, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the total memory consumed by memtables (active and immutable) across different column families in TiKV's RocksDB. Critical for diagnosing memory pressure issues and understanding write buffer memory usage that can impact write performance and trigger flushes.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_engine_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"}\n \n)) by (cf, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 243, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the number of live and obsolete blob files in TiKV's Titan storage engine for large values. Essential for tracking Titan's garbage collection efficiency and identifying potential storage bloat issues when obsolete blob files accumulate excessively.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum((\n tikv_engine_titandb_num_live_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "sum((\n tikv_engine_titandb_num_obsolete_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 244, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the storage space occupied by Titan blob files in TiKV, showing both live (active) and obsolete (garbage) blob file sizes. Essential for tracking Titan storage efficiency, identifying blob file accumulation issues, and determining when garbage collection is needed to reclaim disk space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum((\n tikv_engine_titandb_live_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "sum((\n tikv_engine_titandb_obsolete_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1602, "dashboard": "PD-Cluster-Clinic", "title": "State of all instances", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync state across all PD instances in the cluster. Essential for verifying DR replication status and quickly identifying when instances fall out of sync or experience replication failures.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1501, "dashboard": "PD-Cluster-Clinic", "title": "Syncer index", "group": "Region storage", "used_for": "Used for monitoring the synchronization index of PD's region syncer, which tracks the progress of region metadata synchronization across PD nodes. Essential for diagnosing region information consistency issues and detecting synchronization lag that could affect cluster metadata accuracy.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"sync_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1502, "dashboard": "PD-Cluster-Clinic", "title": "History last index", "group": "Region storage", "used_for": "Used for monitoring the last Raft log index in PD's region synchronization process. Helps diagnose region metadata synchronization issues and detect potential data consistency problems between PD instances in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"last_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 253, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob iter operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of iterator operations (seek, next, prev) on blob files in TiKV's Titan storage engine. Essential for diagnosing Titan blob file scan performance and identifying inefficient iteration patterns that may indicate suboptimal range queries or excessive blob file traversal.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 693, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table checkpoint ts", "group": "Slow Table", "used_for": "Used for monitoring the checkpoint timestamp lag of slow tables in TiCDC changefeeds compared to the current PD timestamp. Helps identify replication delays caused by specific slow tables and diagnose which processing stage (e.g., sorter, mounter) is causing the bottleneck in data synchronization.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 21, "dashboard": "TiDB-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "Used for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for tracking query throughput, identifying traffic patterns, and detecting sudden load changes or anomalies in the cluster's query workload.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1601, "dashboard": "PD-Cluster-Clinic", "title": "State", "group": "DR Autosync", "used_for": "Used for monitoring the current state of PD's disaster recovery (DR) auto-sync replication. Essential for verifying DR replication status and quickly identifying when the cluster transitions between sync states (e.g., sync, async, or disconnected) during disaster recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 247, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Iter touched blob file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the number of blob files accessed during iterator operations in TiKV's Titan storage engine. Helps diagnose performance issues related to large value storage by tracking how many blob files are touched during scans, with higher counts indicating potential read amplification problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 255, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob next duration", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the latency of blob file iterator next operations in TiKV's Titan storage engine. Critical for diagnosing performance issues when scanning large values stored separately in blob files, helping identify slow blob file reads that impact range query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 275, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write delay", "group": "DB", "used_for": "Used for monitoring the write delay in TiCDC's downstream database operations. Essential for diagnosing replication lag issues and identifying bottlenecks in data synchronization to the target database.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 239, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiKV's RocksDB storage engine, tracking maximum, P99, and P95 latencies when writes are temporarily blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when RocksDB flow control mechanisms are throttling write operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 691, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table region count", "group": "Slow Table", "used_for": "Used for monitoring the number of regions in the slowest table being replicated by TiCDC changefeeds. Helps identify replication bottlenecks caused by tables with excessive region counts, which can impact CDC performance and indicate the need for table splitting or workload optimization.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 248, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob key size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob keys in TiKV's Titan storage engine, showing average, P95, and P99 metrics. Helps identify abnormal key size patterns and optimize Titan's large value storage configuration, particularly when diagnosing memory usage or storage efficiency issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 274, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "DB", "used_for": "Used for monitoring the write latency distribution of TiCDC's sorter database operations. Essential for diagnosing performance bottlenecks in TiCDC's data sorting and buffering layer, particularly when replication lag increases or change data capture throughput degrades.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1603, "dashboard": "PD-Cluster-Clinic", "title": "Tick", "group": "DR Autosync", "used_for": "Used for monitoring the rate of DR (Disaster Recovery) auto-sync tick operations in PD. Helps verify that the DR auto-sync mechanism is actively running and processing synchronization cycles, with abnormal rates indicating potential issues with the DR replication coordination.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 689, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Slow Table", "used_for": "Used for identifying which table ID is experiencing the slowest replication in TiCDC changefeeds. Critical for pinpointing specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 272, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Level files", "group": "DB", "used_for": "Used for monitoring the distribution of SSTable files across different levels in TiCDC's embedded database (likely LevelDB/RocksDB). Helps diagnose compaction issues, storage health, and potential performance degradation when files accumulate in higher levels indicating compaction lag.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1605, "dashboard": "PD-Cluster-Clinic", "title": "Recovered region", "group": "DR Autosync", "used_for": "Used for monitoring the number of regions that have been recovered in the DR (Disaster Recovery) auto-sync process. Critical for tracking recovery progress after a disaster event and verifying that data replication has been successfully restored across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recovered_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 699, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag details", "group": "Slow Table", "used_for": "Used for monitoring replication lag details of slow tables in TiCDC changefeeds across different processing stages (resolved TS, checkpoint TS, and TiKV min resolved TS). Critical for diagnosing which specific tables are causing replication delays and identifying bottlenecks in the CDC pipeline stages.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster", "tikv_instance" ], "promql": [ "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 695, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table resolved ts", "group": "Slow Table", "used_for": "Used for monitoring the resolved timestamp progress of slow tables in TiCDC changefeeds compared to the current PD timestamp. Helps identify tables that are lagging behind in replication and diagnose which stage (e.g., puller, sorter, mounter) is causing the slowdown, critical for troubleshooting replication delays.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1604, "dashboard": "PD-Cluster-Clinic", "title": "State ID", "group": "DR Autosync", "used_for": "Used for monitoring the current disaster recovery (DR) state identifier in PD's auto-sync replication. Helps track DR state transitions and verify the cluster's replication status during failover or recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 254, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob seek duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of seek operations in TiKV's Titan blob storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately in Titan's blob files, helping identify slow blob file access patterns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 703, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag duration percentile", "group": "Slow Table", "used_for": "Used for monitoring the 99th percentile replication lag duration of slow tables in TiCDC changefeeds, tracking both resolved timestamp and checkpoint timestamp lag. Critical for identifying tables that are falling behind in data replication and diagnosing TiCDC performance bottlenecks that could impact downstream data freshness.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1606, "dashboard": "PD-Cluster-Clinic", "title": "Recover progress", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) synchronization progress during cluster recovery operations in PD. Essential for tracking the completion status of DR auto-sync recovery and estimating time to full data consistency after a failover event.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recover_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2, "dashboard": "TiDB-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring the commands per second (CPS) executed by each TiDB instance in the cluster. Helps identify uneven query distribution across instances, detect instance-level performance bottlenecks, and troubleshoot load balancing issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 560, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "Scheduler", "used_for": "Used for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scan workload patterns and diagnose potential bottlenecks in TiCDC's data replication process when tracking changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 252, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file discardable ratio distribution", "group": "Titan - $titan_db", "used_for": "Used for monitoring the distribution of discardable data ratios across Titan blob files in TiKV's storage engine. Helps identify garbage collection opportunities and assess storage space efficiency by showing how much obsolete data exists in blob files that can be reclaimed through GC.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum((\n tikv_engine_titandb_blob_file_discardable_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (ratio, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 250, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob retrieval operations in TiKV's Titan storage engine. Essential for tracking how frequently large values are being read from separate blob files, helping diagnose performance issues related to Titan's key-value separation architecture.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 251, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob retrieval operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately from the LSM tree in Titan-enabled instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 249, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob value size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob values stored in TiKV's Titan storage engine, showing average, P95, and P99 metrics. Essential for understanding large value storage patterns and optimizing Titan's blob file management when dealing with values exceeding the min-blob-size threshold.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 277, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write bytes", "group": "DB", "used_for": "Used for monitoring the rate of bytes written to TiCDC's sorter database storage. Helps diagnose I/O bottlenecks and storage write performance issues in the change data capture pipeline, particularly when replication lag occurs.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 273, "dashboard": "Prometheus-TiCDC-Clinic", "title": "In-memory data size", "group": "DB", "used_for": "Used for monitoring the amount of data held in memory by TiCDC's sorter component per instance. Critical for diagnosing memory pressure issues and determining if TiCDC needs memory tuning or if data is being efficiently spilled to disk during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 288, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write speed", "group": "DB", "used_for": "Used for monitoring the write throughput of TiCDC's sorter database, which temporarily stores change data during replication. Essential for diagnosing TiCDC performance bottlenecks, identifying instances with high disk I/O pressure, and ensuring the sorter can keep up with upstream data changes.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 137, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per minute in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing which types of errors (e.g., syntax errors, permission issues, or execution timeouts) are affecting application queries.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 267, "dashboard": "TiDB-Cluster-Clinic", "title": "Affected Rows By Type", "group": "Query Summary", "used_for": "Used for monitoring the rate of rows affected by different SQL statement types (INSERT, UPDATE, DELETE, etc.) in TiDB. Helps identify which operations are modifying the most data and diagnose unexpected data change patterns or bulk operation impacts on the cluster.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type)", "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 42, "dashboard": "TiDB-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (success/error). Essential for tracking overall query throughput, comparing current performance against historical baselines (1-day offset), and quickly identifying sudden drops in successful queries or spikes in failed commands that indicate cluster health issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 258, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file read duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring read latency when accessing Titan blob files in TiKV's storage engine. Critical for diagnosing performance issues with large value storage, showing average, P95, and P99 read durations to identify slow blob file access patterns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 257, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob keys flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob key operations in TiKV's Titan storage engine, tracking both keys written to and read from blob files. Essential for understanding Titan's key-value separation behavior and diagnosing issues with large value storage performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 259, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob bytes flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob data flow in TiKV's Titan storage engine, tracking both bytes written and bytes read. Essential for understanding Titan's large value storage patterns and diagnosing performance issues related to blob file operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 285, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Compact duration", "group": "DB", "used_for": "Used for monitoring the compaction duration of TiCDC's sorter database, which manages event ordering during data replication. Essential for diagnosing TiCDC performance degradation caused by slow database compaction operations that can impact replication latency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 112, "dashboard": "TiDB-Cluster-Clinic", "title": "Slow query", "group": "Query Summary", "used_for": "Used for monitoring the 90th percentile latency breakdown of slow queries in TiDB, including total processing time, coprocessor execution time, and wait time. Essential for diagnosing performance bottlenecks in slow queries and identifying whether delays occur during query processing, storage layer operations, or resource waiting.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 271, "dashboard": "Prometheus-TiCDC-Clinic", "title": "On disk data size", "group": "DB", "used_for": "Used for monitoring the total size of data stored on disk by TiCDC's sorter component per instance. Essential for tracking disk space consumption during change data capture operations and identifying potential disk space issues or memory overflow scenarios where data is spilled to disk.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 246, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache hit", "group": "Titan - $titan_db", "used_for": "Used for monitoring the cache hit rate of Titan's blob cache in TiKV. Helps diagnose whether large values are being efficiently cached in memory, with low hit rates indicating potential performance issues due to excessive disk reads for blob data.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "(sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 260, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file write duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring write latency to Titan blob files in TiKV's storage engine, showing average, P99, and P95 durations. Critical for diagnosing performance issues when large values are stored separately in Titan's blob storage, helping identify write bottlenecks in the value separation architecture.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 218, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Idle Duration", "group": "Query Summary", "used_for": "Used for monitoring the idle time duration of database connections in TiDB, separated by transaction state (in_txn='1' for active transactions, in_txn='0' for idle connections). Essential for identifying connection pool inefficiencies, detecting long-running idle transactions that may hold locks, and diagnosing resource waste from improperly managed connections.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 279, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write OPS", "group": "DB", "used_for": "Used for monitoring the write operations per second to TiCDC's sorter database across instances. Essential for tracking TiCDC's data processing throughput and identifying instances with abnormal write activity or performance bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 136, "dashboard": "TiDB-Cluster-Clinic", "title": "999 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile (P999) query execution duration in TiDB, broken down by SQL statement type. Critical for identifying tail latency issues and detecting slow query outliers that affect the worst-case user experience, helping diagnose performance degradation before it impacts the majority of queries.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 278, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - First", "group": "DB", "used_for": "Used for monitoring the latency of the first read operation in TiCDC's sorter database iterator. Helps diagnose performance issues in TiCDC's data sorting mechanism, particularly when investigating delays in change data capture processing or replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 261, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file sync operations in TiKV's Titan storage engine. Essential for diagnosing I/O performance issues and understanding the frequency of data persistence operations when using Titan for large value storage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - Next", "group": "DB", "used_for": "Used for monitoring the latency distribution of iterator \"next\" operations when reading from TiCDC's sorter database. Critical for diagnosing performance bottlenecks in TiCDC's event sorting and processing pipeline, especially when replication lag increases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 262, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC action", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of Titan blob garbage collection actions in TiKV's storage engine. Helps diagnose Titan GC efficiency and identify issues with blob file cleanup, which is critical for managing disk space usage in large value workloads.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_gc_action_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 132, "dashboard": "TiDB-Cluster-Clinic", "title": "95 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation affecting most queries and establishing baseline latency expectations for different workload patterns.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 134, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution latency in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation and diagnosing slow query issues affecting the majority of database operations.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 130, "dashboard": "TiDB-Cluster-Clinic", "title": "80 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution latency in TiDB, showing the duration threshold that 80% of queries complete under. Helps identify typical query performance and detect degradation affecting the majority of workload before it impacts tail latencies.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 280, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - First", "group": "DB", "used_for": "Used for monitoring the rate of first-read operations per second in TiCDC's sorter database iterator. Helps identify throughput patterns and potential bottlenecks when TiCDC initializes data reads during change data capture processing.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 264, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the garbage collection latency in TiKV's Titan storage engine, which manages large values stored separately from RocksDB. Critical for diagnosing performance issues related to Titan blob file cleanup, showing average, P95, and P99 durations to identify GC bottlenecks that could impact write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed count", "group": "KVClient", "used_for": "Used for monitoring the number of active EventFeed gRPC connections between TiCDC and TiKV nodes. Essential for diagnosing change data capture issues, detecting connection instabilities, and ensuring TiCDC is properly receiving data change events from all TiKV instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)", "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 263, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob file sync operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing write performance issues when large values are stored separately in blob files, as high sync latency directly impacts write throughput and transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 283, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - Next", "group": "DB", "used_for": "Used for monitoring the rate of iterator \"next\" read operations in TiCDC's sorter database per instance. Helps identify sorter read throughput patterns and diagnose potential bottlenecks in TiCDC's data sorting and processing pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC bytes flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of data flow during Titan blob file garbage collection in TiKV's storage engine. Helps track GC efficiency and identify potential issues with large value cleanup, which is critical for managing disk space when using Titan for storing large key-values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 271, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "OPS", "group": "In Memory Engine", "used_for": "Used for monitoring the operations per second (OPS) throughput of TiKV's In-Memory Engine across different operation types. Essential for tracking in-memory cache performance and identifying bottlenecks or unusual activity patterns in memory-based key-value operations.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_kv_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 265, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC keys flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of key data flow during Titan blob garbage collection in TiKV's storage engine. Helps diagnose Titan GC efficiency and identify issues with blob file cleanup by tracking how many keys are being processed during garbage collection operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM Detail", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the per-minute increase in query execution errors across TiDB instances. Essential for identifying which specific TiDB servers are experiencing query failures and diagnosing application-level errors, connection issues, or SQL syntax problems.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 80 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution duration across individual TiDB instances. Helps identify performance variations between instances and pinpoint specific TiDB nodes experiencing elevated query latency, useful for load balancing analysis and instance-level troubleshooting.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 81, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 999 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile query execution duration for each individual TiDB instance. Essential for identifying specific TiDB nodes experiencing tail latency issues or performance degradation, enabling targeted troubleshooting of slow query problems at the instance level.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 29, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client receive events/s", "group": "KVClient", "used_for": "**Used For:**\n\nUsed for monitoring the rate of change events received by TiCDC's KV client from TiKV per second, broken down by instance, event type, and changefeed. Essential for tracking data capture throughput and diagnosing replication lag or event processing bottlenecks in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, type, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 266, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC input file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size of input files processed during Titan blob garbage collection operations in TiKV. Helps diagnose Titan storage engine GC efficiency and identify potential issues with large value cleanup, showing average, P95, and P99 file sizes to detect outliers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 25, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution duration for each TiDB instance. Helps identify performance bottlenecks and uneven load distribution across individual TiDB servers by comparing p99 latencies between instances.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 188, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client gRPC stream count", "group": "KVClient", "used_for": "Used for monitoring the number of active gRPC streams between TiCDC's KV client and each TiKV store. Essential for diagnosing connection issues, detecting stream leaks, and understanding TiCDC's data replication connectivity health across the cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 269, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file garbage collection operations in TiKV's Titan storage engine. Essential for tracking Titan GC activity and diagnosing issues with blob file cleanup, which impacts disk space management and storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(\n tikv_engine_blob_gc_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 177, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client scanning regions", "group": "KVClient", "used_for": "Used for monitoring the number of regions currently being scanned by TiCDC's KV client during change data capture operations. Helps diagnose CDC performance issues related to region scanning overhead and identify instances with excessive region scan activity that may impact replication lag.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 459, "dashboard": "Prometheus-TiCDC-Clinic", "title": "RegionCache operation/s", "group": "KVClient", "used_for": "Used for monitoring the rate of TiCDC region cache operations per second, broken down by operation type. Essential for diagnosing region metadata access patterns and identifying potential bottlenecks in TiCDC's interaction with TiKV's region cache during change data capture.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 97, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client batch resolved region count", "group": "KVClient", "used_for": "Used for monitoring the distribution of batch sizes when TiCDC's KV client resolves region events. Helps diagnose event batching efficiency and identify potential bottlenecks in region change stream processing.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 28, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "KVClient", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's event feed per minute, categorized by error type. Critical for diagnosing data replication issues and identifying whether problems stem from TiCDC's KV client connections or underlying PD scheduling operations like leader transfers and region/peer changes.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 95 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration for each TiDB instance. Helps identify performance variations across individual TiDB servers and pinpoint specific instances experiencing elevated query latency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 17, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Event size percentile", "group": "KVClient", "used_for": "Used for monitoring the distribution of event sizes in TiCDC's KV client, tracking how large change data events are being processed. Helps identify abnormally large events that could impact replication performance and diagnose memory or throughput bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_event_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client dispatch events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of KV events dispatched by TiCDC's KV client, including event send counts and batch resolved event sizes per second. Essential for tracking TiCDC's event processing throughput and identifying bottlenecks in change data capture pipelines when replication lag occurs.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, namespace,changefeed, type)", "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace, changefeed, table)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572003, "dashboard": "TiDB-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Essential for tracking server stability, identifying recent restarts or crashes, and verifying cluster availability during troubleshooting or maintenance operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 251, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client cached regions", "group": "KVClient", "used_for": "Used for monitoring the number of TiKV regions cached by TiCDC's KV client for each changefeed instance. Helps diagnose region cache efficiency and potential memory issues when the cached region count grows abnormally high or drops unexpectedly during data replication.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 460, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backoff percentiles", "group": "KVClient", "used_for": "Used for monitoring the 99.999th percentile backoff duration when TiCDC's KV client encounters retryable errors communicating with TiKV. Critical for identifying extreme cases of replication delays caused by backoff mechanisms during transient failures or resource contention.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data change capture throughput and identifying bottlenecks in the event collection pipeline from TiKV to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 272, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries in Multi-Statement", "group": "Query Detail", "used_for": "Used for monitoring the average number of queries contained within multi-statement requests and the total rate of queries in multi-statement batches. Helps identify applications using batch query patterns and diagnose potential performance issues related to multi-statement execution overhead.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/sum(rate(tidb_server_multi_query_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 268, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC output file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size of output files generated during Titan blob garbage collection operations in TiKV. Helps assess GC efficiency and identify potential issues with blob file compaction, particularly when output file sizes are unexpectedly large or show unusual patterns across average, P95, and P99 percentiles.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"}\n \n)) by ($additional_groupby) ", "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"}\n \n)) by ($additional_groupby) ", "max((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 278, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Count", "group": "In Memory Engine", "used_for": "Used for monitoring the number of regions cached in TiKV's in-memory engine across instances. Helps identify memory usage patterns and diagnose issues related to region cache capacity or uneven distribution of cached regions across TiKV nodes.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_in_memory_engine_cache_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 276, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Cache Miss Reason", "group": "In Memory Engine", "used_for": "Used for tracking why region cache lookups fail in TiKV's in-memory engine, broken down by failure reason and instance. Essential for diagnosing memory engine cache efficiency issues and understanding whether misses are due to eviction, invalidation, or other causes that impact read performance.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 184, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB Server Status", "group": "Server", "used_for": "Used for monitoring the availability status of TiDB server instances in the cluster. Shows the count of healthy (up) versus unhealthy (down) TiDB nodes, helping quickly identify server failures or connectivity issues that could impact query processing.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} == 1)", "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} == 0)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 277, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Usage", "group": "In Memory Engine", "used_for": "Used for monitoring memory consumption of TiKV's In-Memory Engine across instances. Essential for tracking memory usage patterns, detecting memory leaks, and ensuring the in-memory engine stays within allocated memory limits to prevent OOM issues.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_in_memory_engine_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimate remaining time for initialization", "group": "KVClient", "used_for": "Used for estimating how much time remains for TiCDC's initial region cache population during changefeed startup. Helps operators understand initialization progress and predict when the changefeed will be ready to begin replicating data.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 168, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring CPU utilization of TiDB server instances compared to their maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiDB instances need vertical scaling or load redistribution.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 188, "dashboard": "TiDB-Cluster-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of open file descriptors in TiDB server processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause service disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 280, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region GC Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the garbage collection duration for regions in TiKV's In-Memory Engine. Helps diagnose memory management performance issues and identify when GC operations are taking longer than expected, which could impact overall TiKV performance.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_gc_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 292, "dashboard": "TiDB-Cluster-Clinic", "title": "Runtime GC rate and GOMEMLIMIT", "group": "Server", "used_for": "Used for monitoring TiDB's Go garbage collection frequency and memory limit configuration. Helps diagnose memory pressure issues and evaluate whether GOMEMLIMIT settings are appropriate when experiencing high GC overhead or memory-related performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 279, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC Filter", "group": "In Memory Engine", "used_for": "Used for monitoring the rate of garbage collection filtering operations in TiKV's in-memory engine, categorized by filter type. Helps diagnose memory management efficiency and identify potential issues with stale data cleanup in the in-memory storage layer.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_gc_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 453, "dashboard": "Prometheus-TiCDC-Clinic", "title": "ResolveLock count/m", "group": "KVClient", "used_for": "Used for monitoring the rate of lock resolution operations per minute in TiCDC's KV client. Essential for diagnosing transaction conflicts and lock contention issues that may impact CDC replication latency, with breakdowns by resolution type to identify specific lock handling patterns.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "Used for monitoring the number of active client connections to TiDB instances and tracking connection token usage. Essential for diagnosing connection exhaustion issues, identifying connection leaks, and determining if the cluster is approaching connection limits that could cause new connection failures.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 49, "dashboard": "TiDB-Cluster-Clinic", "title": "Events OPM", "group": "Server", "used_for": "Used for monitoring the rate of server events occurring in TiDB instances, measured in operations per minute over a 10-minute window. Helps identify abnormal event spikes, server state changes, or unusual activity patterns that may indicate system issues or configuration changes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 99, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event sorting operations in TiCDC's puller component. Essential for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being processed.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 275, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Cache Hit Rate", "group": "In Memory Engine", "used_for": "Used for monitoring the cache hit rate of region data in TiKV's In-Memory Engine. Helps identify memory cache effectiveness and diagnose performance issues when queries frequently miss the region cache, requiring slower disk reads.", "component": "TIKV", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 281, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Load Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time taken to load regions into TiKV's in-memory engine. Critical for diagnosing slow region initialization and identifying performance bottlenecks during region loading operations, especially after restarts or region migrations.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_load_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 272, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read MBps", "group": "In Memory Engine", "used_for": "Used for monitoring the read throughput (in megabytes per second) of TiKV's in-memory engine compared to the regular storage engine. Helps identify memory cache effectiveness and diagnose whether read operations are being served from memory or disk, which is critical for understanding read performance bottlenecks.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 205, "dashboard": "TiDB-Cluster-Clinic", "title": "Disconnection Count", "group": "Server", "used_for": "Used for tracking the total number of client disconnections from TiDB instances, categorized by disconnection result type. Essential for diagnosing connection stability issues, identifying abnormal client disconnection patterns, and troubleshooting network or client-side problems that may impact application availability.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 283, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Eviction Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time taken to evict regions from TiKV's in-memory engine. Critical for diagnosing memory pressure issues and understanding the performance impact of region evictions when the in-memory engine reaches capacity limits.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_eviction_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Load Count", "group": "In Memory Engine", "used_for": "Used for monitoring the rate of region loading operations into TiKV's in-memory engine across instances. Helps identify memory engine initialization patterns and diagnose issues with region data loading from persistent storage into memory.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_in_memory_engine_load_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "TiDB-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiDB instances. Essential for detecting goroutine leaks, resource exhaustion issues, and identifying abnormal concurrency patterns that could indicate connection pooling problems or stuck operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_sched_goroutines_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\"tidb.*\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 105, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event merge operations in TiCDC's sorter component. Critical for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being processed.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 165, "dashboard": "TiDB-Cluster-Clinic", "title": "Prepare Statement Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of prepared statements currently cached in TiDB instances. Helps identify potential memory pressure from excessive prepared statement accumulation and diagnose connection pooling or statement lifecycle management issues in applications.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 707, "dashboard": "Prometheus-TiCDC-Clinic", "title": "mounter group channel size", "group": "KVClient", "used_for": "Used for monitoring the input channel size of TiCDC's mounter group, which processes change data from TiKV. Helps identify backpressure issues when the channel accumulates too many pending events, indicating the mounter cannot keep up with incoming data changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 274, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Cache Hit", "group": "In Memory Engine", "used_for": "Used for monitoring the hit rate of region cache in TiKV's In-Memory Engine, showing how frequently data requests are served directly from memory rather than disk. Essential for evaluating memory cache effectiveness and identifying opportunities to optimize read performance by improving cache utilization.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 284, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Eviction Count", "group": "In Memory Engine", "used_for": "Used for monitoring the number of region evictions from TiKV's in-memory engine, categorized by eviction type. Critical for diagnosing memory pressure issues and understanding when regions are being evicted from cache to free up memory resources.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_in_memory_engine_eviction_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 287, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Prepare for write duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time TiKV's in-memory engine spends preparing for write operations. Critical for diagnosing write path performance issues and identifying bottlenecks in the in-memory engine's write preparation phase.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "TiDB-Cluster-Clinic", "title": "Panic And Critial Error", "group": "Server", "used_for": "Used for monitoring panic events and critical errors occurring in TiDB server instances. Essential for immediately detecting severe system failures and stability issues that require urgent investigation and intervention.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 68, "dashboard": "TiDB-Cluster-Clinic", "title": "Internal SQL OPS", "group": "Query Detail", "used_for": "Used for monitoring the throughput of internal SQL operations executed by TiDB, such as statistics collection, auto-analyze, and system maintenance queries. Essential for distinguishing internal database overhead from user workload and diagnosing performance issues caused by background tasks.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 106, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting and merging operations in TiCDC's puller component. Critical for diagnosing replication lag issues caused by slow DDL event processing, especially when changefeeds experience delays in synchronizing schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 285, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "In Memory Engine", "used_for": "Used for monitoring write operation latency in TiKV's In-Memory Engine. Essential for diagnosing performance bottlenecks when data is being written to the in-memory storage layer, helping identify if memory-based operations are experiencing unexpected delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 267, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the latency distribution of TiCDC's etcd worker tick reactor operations. Essential for diagnosing performance issues in TiCDC's etcd interaction layer, particularly when experiencing delays in metadata synchronization or coordination tasks.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 289, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd MVCC DB total size", "group": "Etcd", "used_for": "Used for monitoring the total size of PD's embedded etcd MVCC database in bytes. Critical for tracking metadata storage growth and identifying potential disk space issues or excessive metadata accumulation that could impact cluster stability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 191, "dashboard": "TiDB-Cluster-Clinic", "title": "Skip Binlog Count", "group": "Server", "used_for": "Used for monitoring the count of critical errors related to binlog skipping in TiDB instances. Essential for identifying binlog replication failures or data consistency issues when TiDB is configured with binlog enabled for downstream synchronization.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 286, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% In-memory engine write duration per server", "group": "In Memory Engine", "used_for": "Used for monitoring the 99th and 99.99th percentile write latency to TiKV's in-memory engine on each server instance. Critical for identifying write performance bottlenecks and tail latency issues in the in-memory storage layer, helping diagnose whether specific TiKV nodes are experiencing degraded write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 288, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% In-memory engine prepare for write duration per server", "group": "In Memory Engine", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of prepare-for-write operations in TiKV's in-memory engine, broken down by server instance. Critical for diagnosing write path performance issues and identifying instances experiencing slow in-memory engine preparation phases that could impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 211, "dashboard": "TiDB-Cluster-Clinic", "title": "Client Data Traffic", "group": "Server", "used_for": "Used for monitoring the network traffic volume between TiDB servers and clients, broken down by traffic type (inbound/outbound). Essential for identifying network bottlenecks, unusual traffic patterns, and diagnosing connectivity issues or data transfer anomalies.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 82, "dashboard": "TiDB-Cluster-Clinic", "title": "Keep Alive OPM", "group": "Server", "used_for": "Used for monitoring the rate of keep-alive operations per minute sent by TiDB instances to maintain connections with clients or other components. Helps diagnose connection stability issues, detect abnormal connection churn, or identify instances experiencing connectivity problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 290, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "In Memory Engine", "used_for": "Used for monitoring the P95 and P99 latency of seek operations in TiKV's In-Memory Engine. Critical for diagnosing performance issues with range scans and point lookups in the memory storage layer, especially when investigating slow read queries or memory engine bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.95,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_seek_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_seek_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 264, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the p99 and p999 latency of TiCDC's etcd worker tick reactor operations. Essential for diagnosing TiCDC replication delays caused by slow etcd interactions, particularly when investigating change feed lag or metadata synchronization issues.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 111, "dashboard": "TiDB-Cluster-Clinic", "title": "Get Token Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of token acquisition in TiDB's connection rate limiting mechanism. Critical for diagnosing connection throttling issues and identifying when clients experience delays due to token bucket rate limiting during high concurrency scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3, "dashboard": "TiDB-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiDB server memory consumption, including process resident memory and Go heap allocation. Essential for diagnosing memory leaks, OOM risks, and optimizing memory configuration when TiDB instances experience high memory pressure or crashes.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_gc_heap_goal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}) by (module, instance)", "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10039, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client slow initalize region count", "group": "KVClient", "used_for": "Used for monitoring the number of regions in TiCDC's KV client that are experiencing slow initialization. Helps diagnose replication lag issues caused by delayed region setup, which can indicate network problems, TiKV overload, or region scheduling bottlenecks affecting CDC's ability to capture changes efficiently.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_kvclient_slow_initialize_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 114, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd health check duration", "group": "Etcd", "used_for": "Used for monitoring the latency of TiCDC's health check operations against etcd at p95, p99, and p999 percentiles. Critical for diagnosing etcd connectivity issues and detecting when TiCDC's coordination layer experiences slow responses, which can impact change data capture reliability and cluster metadata operations.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 167, "dashboard": "TiDB-Cluster-Clinic", "title": "Handshake Error OPS", "group": "Server", "used_for": "Used for monitoring the rate of client connection handshake failures per TiDB instance. Essential for diagnosing authentication issues, SSL/TLS configuration problems, or client compatibility issues that prevent successful database connections.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 291, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Oldest Auto GC SafePoint", "group": "In Memory Engine", "used_for": "Used for monitoring the oldest automatic garbage collection safe point in TiKV's in-memory engine. Helps identify potential memory pressure issues and diagnose when old data versions are not being cleaned up properly, which could lead to memory bloat.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 254, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction sizes in TiCDC's etcd worker component. Helps identify abnormally large transactions that could impact etcd performance and diagnose potential issues with metadata operations or state synchronization.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 292, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Newest Auto GC SafePoint", "group": "In Memory Engine", "used_for": "Used for monitoring the most recent garbage collection safe point in TiKV's in-memory engine. Helps track memory cleanup progress and diagnose issues where stale data isn't being properly garbage collected, which could lead to memory bloat.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 53, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting operations in TiCDC's puller entry sorter. Critical for diagnosing replication lag issues caused by slow DDL processing, especially when changefeeds experience delays in handling schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 171, "dashboard": "TiDB-Cluster-Clinic", "title": "RCCheckTS WriteConflict Num", "group": "Server", "used_for": "Used for monitoring the number of write conflicts detected during read-committed isolation level timestamp checks in TiDB. Essential for diagnosing transaction conflicts and concurrency issues that may cause transaction retries or failures in read-committed workloads.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_rc_check_ts_conflict_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 289, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Iterator operations", "group": "In Memory Engine", "used_for": "Used for monitoring iterator operation rates in TiKV's In-Memory Engine, tracking seek operations, successful seeks, and next operations. Essential for diagnosing memory engine scan performance and understanding how efficiently the in-memory storage layer is handling range queries and data traversal.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 256, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob prev duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob file backward iteration operations in TiKV's Titan storage engine. Essential for diagnosing performance issues when scanning or reading blob values in reverse order, particularly useful when investigating slow range queries or reverse scans involving large values stored in Titan.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "max((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 258, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration percentile", "group": "Etcd", "used_for": "Used for monitoring the p95 and p99 latency of etcd transaction executions in TiCDC's etcd worker. Critical for diagnosing metadata operation performance issues and identifying when etcd transactions become slow, which can impact TiCDC's ability to coordinate change data capture tasks and maintain cluster state.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 260, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size percentile", "group": "Etcd", "used_for": "Used for monitoring the 95th and 99th percentile transaction size in bytes for TiCDC's etcd worker operations. Essential for identifying abnormally large etcd transactions that could impact TiCDC's metadata management performance and detecting potential issues with change data capture coordination overhead.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 256, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction execution duration in TiCDC's etcd worker. Critical for diagnosing coordination and metadata operation latency issues that could impact TiCDC's change data capture performance and replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 296, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler flow", "group": "Flow Control", "used_for": "Used for monitoring write flow rates through TiKV's scheduler and identifying when flow control throttling is active. Essential for diagnosing write performance degradation caused by flow control mechanisms limiting throughput to prevent system overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Statement Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of statement counts per transaction in TiDB. Helps identify transactions with excessive statements that may cause performance issues, long-running transactions, or inefficient batch operations requiring optimization.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 290, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% Handle transactions duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in TiCDC's PD component. Critical for identifying slow etcd operations that could impact TiCDC's metadata management and coordination, helping diagnose replication delays caused by etcd performance bottlenecks.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572016, "dashboard": "TiDB-Cluster-Clinic", "title": "Internal Sessions", "group": "Server", "used_for": "Used for monitoring the number of internal sessions in TiDB, which are background connections used for system operations like statistics collection, auto-analyze, and internal queries. Helps identify resource consumption from internal processes and diagnose issues when internal sessions accumulate abnormally or impact cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_internal_sessions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 293, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Auto GC SafePoint Gap", "group": "In Memory Engine", "used_for": "Used for monitoring the gap between the newest and oldest safe points in TiKV's in-memory engine, measured in time units. Critical for detecting potential memory pressure issues and ensuring proper garbage collection of in-memory data, as a large gap may indicate GC is falling behind or memory is not being reclaimed efficiently.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 297, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler discard ratio", "group": "Flow Control", "used_for": "Used for monitoring the ratio of discarded scheduler commands in TiKV due to flow control mechanisms. Helps identify when TiKV is rejecting requests because of resource pressure or I/O bottlenecks, indicating potential overload situations that require capacity adjustment or optimization.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 294, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Auto GC SafePoint Gap With TiKV", "group": "In Memory Engine", "used_for": "Used for monitoring the gap between TiDB's GC safe point and TiKV's in-memory engine safe point, measured in units of 2^18. Critical for detecting potential memory pressure issues when the gap grows too large, indicating the in-memory engine may be holding onto old MVCC versions that cannot be garbage collected.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring transaction operations per second in TiDB, broken down by transaction type (commit/rollback) and mode (optimistic/pessimistic), including both general user transactions and internal system transactions. Essential for tracking transaction throughput, identifying transaction pattern changes, and diagnosing issues related to transaction processing capacity or abnormal commit/rollback ratios.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (type, txn_mode)", "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"internal\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 72, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile transaction duration in TiDB, broken down by transaction type (optimistic/pessimistic) and showing both percentile and average latencies. Essential for identifying transaction performance degradation, comparing transaction modes, and diagnosing slow commit issues that impact application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, txn_mode))", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 67, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction retry attempts in TiDB. Helps diagnose transaction conflicts, write contention issues, and identify workloads experiencing frequent optimistic lock failures that may benefit from pessimistic locking or application-level optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 298, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Throttle duration", "group": "Flow Control", "used_for": "Used for monitoring the cumulative time TiKV scheduler operations spend in throttled state due to flow control mechanisms. Essential for diagnosing write performance degradation caused by flow control throttling, helping identify when the system is actively limiting request rates to prevent overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 300, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller actions", "group": "Flow Control", "used_for": "Used for monitoring the rate of flow control actions triggered by TiKV's scheduler throttling mechanism. Essential for diagnosing when and why TiKV is applying backpressure to writes, helping identify storage bottlenecks and I/O saturation issues across different column families.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC network traffic", "group": "TiKV", "used_for": "Used for monitoring the network traffic sent by TiKV's CDC (Change Data Capture) gRPC messages, broken down by instance and message type. Essential for diagnosing CDC replication bottlenecks, network bandwidth issues, and identifying which TiKV instances are generating high CDC traffic during data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 299, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler throttled CF", "group": "Flow Control", "used_for": "Used for monitoring which column families (CFs) in TiKV's scheduler are being throttled due to flow control mechanisms. Helps identify write pressure and resource contention issues when the scheduler actively limits operations on specific column families to prevent overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 147, "dashboard": "Prometheus-TiCDC-Clinic", "title": "gRPC message count", "group": "TiKV", "used_for": "Used for monitoring the rate of gRPC message requests handled by TiKV instances, broken down by message type. Essential for identifying which gRPC operations are most active and diagnosing communication bottlenecks between TiCDC and TiKV during data replication.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 60, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC CPU", "group": "TiKV", "used_for": "Used for monitoring CPU usage of TiCDC-related threads running on TiKV instances, including CDC workers, CDC event processors, and TSO threads. Essential for diagnosing CDC performance bottlenecks, identifying instances with high replication overhead, and determining if CDC workloads are impacting TiKV's overall performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 196, "dashboard": "TiDB-Cluster-Clinic", "title": "Commit Token Wait Duration", "group": "Transaction", "used_for": "Used for monitoring the wait time transactions spend acquiring commit tokens in TiDB's batch executor at p80, p95, and p99 percentiles. Critical for diagnosing transaction commit bottlenecks caused by token contention when concurrent commits exceed available token capacity, which can indicate the need to adjust `committer-concurrency` settings.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring the throughput of KV transaction operations in TiDB, broken down by operation type (commit, rollback, etc.) per instance. Essential for identifying transaction bottlenecks, understanding workload patterns, and diagnosing issues when transaction processing capacity becomes saturated.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC memory", "group": "TiKV", "used_for": "Used for monitoring CDC (Change Data Capture) related memory usage in TiKV instances. Helps diagnose memory pressure issues by tracking CDC sink memory, old value cache, and overall memory consumption excluding block cache, essential for troubleshooting CDC replication performance and memory leaks.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 44, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Regions Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of region counts involved in transaction prewrite operations. Helps identify transactions that span many regions, which can indicate cross-region transaction overhead and potential performance bottlenecks in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 301, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush/L0 flow", "group": "Flow Control", "used_for": "**Used For:**\n\nUsed for monitoring the flow control metrics related to L0 SST files and memtable flush operations in TiKV's RocksDB storage engine. Essential for diagnosing write stalls and understanding when flow control mechanisms are throttling writes due to excessive L0 files or pending flush operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "TiDB-Cluster-Clinic", "title": "Session Retry Error OPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate of transaction retry errors per second in TiDB, categorized by error type and SQL type. Essential for diagnosing transaction conflicts, write-write conflicts, and optimistic transaction failures that require automatic retry or client intervention.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, sql_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 153, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved Region ID", "group": "TiKV", "used_for": "Used for identifying which TiKV region has the minimum resolved timestamp in CDC replication, helping diagnose replication lag issues. Critical for pinpointing specific regions that are blocking CDC progress and causing replication delays across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 302, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller factors", "group": "Flow Control", "used_for": "Used for monitoring the key factors that trigger TiKV's flow control mechanism, including L0 SST file count, memtable count, and average L0 levels across instances. Essential for diagnosing write throttling issues and understanding when TiKV applies backpressure to prevent storage engine overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 303, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "Flow Control", "used_for": "Used for monitoring the amount of data pending compaction in TiKV's RocksDB storage engine across different column families. Critical for diagnosing write stalls and flow control issues, as excessive pending compaction bytes indicate the storage engine cannot keep up with write pressure, potentially triggering write throttling to prevent system overload.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Incremental scan duration", "group": "TiKV", "used_for": "Used for monitoring the latency distribution of incremental data scanning operations in TiKV for CDC (Change Data Capture). Critical for diagnosing CDC replication lag issues and identifying performance bottlenecks in the data capture pipeline from TiKV to downstream systems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 193, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution of KV transaction operations in TiDB at the 80th, 95th, and 99th percentiles. Essential for diagnosing transaction performance issues, identifying slow transaction types (commit, prewrite, etc.), and detecting latency degradation that impacts application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 209, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of key-value pairs written per transaction in TiDB. Helps identify transactions with abnormally high write volumes that may cause performance bottlenecks or exceed transaction size limits.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 152, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved ts", "group": "TiKV", "used_for": "Used for monitoring the replication lag between TiKV's minimum resolved timestamp and the current cluster TSO (Timestamp Oracle) in TiCDC data capture scenarios. Critical for identifying CDC replication delays and diagnosing whether specific TiKV instances are falling behind in change data capture processing.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 72, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Incremental scan duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.99th percentile latency of TiCDC incremental scan operations in TiKV. Critical for identifying extreme tail latency in change data capture scanning, helping diagnose performance bottlenecks that could delay data replication to downstream systems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))", "sum(rate(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance) / sum(rate(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 273, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor Handle duration", "group": "In Memory Engine", "used_for": "Used for monitoring the latency of coprocessor request handling in TiKV's in-memory engine, tracking p99 and p9999 percentiles. Critical for diagnosing slow coprocessor operations (table scans, aggregations, index lookups) and identifying tail latency issues that impact analytical query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 166, "dashboard": "TiDB-Cluster-Clinic", "title": "Time Jump Back OPS", "group": "Server", "used_for": "Used for detecting system clock backward jumps on TiDB instances, measured as occurrences per second. Critical for identifying time synchronization issues that can cause transaction timestamp conflicts, data consistency problems, and TSO (Timestamp Oracle) anomalies in distributed transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 70, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Resolved ts lag duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.999th percentile lag between TiKV's resolved timestamp and current time in TiCDC replication. Critical for identifying extreme tail latency in change data capture, helping diagnose replication delays and ensuring data consistency in downstream systems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 33, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative count of key-value pairs written per transaction in TiDB. Helps identify transaction write patterns, detect abnormal batch sizes, and diagnose issues related to large transactions that may impact cluster performance or cause write hotspots.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 201, "dashboard": "TiDB-Cluster-Clinic", "title": "Statement Lock Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of lock keys acquired per statement in TiDB transactions. Helps diagnose lock contention issues and identify statements that acquire excessive locks, which can lead to transaction conflicts and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 309, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands queued in each TiKV instance. Helps identify scheduler bottlenecks and command processing delays that could indicate resource contention or performance degradation in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 304, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time transactions spend being throttled due to flow control mechanisms in TiKV. Critical for diagnosing performance degradation caused by resource pressure, helping identify when and which transaction types are being rate-limited to protect cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 197, "dashboard": "TiDB-Cluster-Clinic", "title": "Acquire Pessimistic Locks Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the latency of acquiring pessimistic locks during transactions at p80, p95, and p99 percentiles. Critical for diagnosing transaction performance issues, lock contention problems, and identifying when pessimistic locking operations are causing delays in write-heavy workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "TiDB-Cluster-Clinic", "title": "Send HeartBeat Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of transaction heartbeat messages sent from TiDB to TiKV to keep pessimistic locks alive. Critical for diagnosing transaction timeout issues and lock maintenance problems in long-running transactions, especially when locks are held beyond the default TTL.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 291, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% WAL fsync duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile fsync duration of etcd's Write-Ahead Log (WAL) in TiCDC. Critical for diagnosing disk I/O performance issues that could impact TiCDC's metadata operations and change data capture reliability, as high WAL fsync latency indicates slow disk writes affecting etcd's consistency guarantees.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 210, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction write sizes in TiDB, showing how many bytes are written per transaction. Helps identify large transactions that may cause performance issues, memory pressure, or exceed transaction size limits, and is essential for diagnosing write-heavy workload patterns.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 140, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Incremental scan tasks count", "group": "TiKV", "used_for": "Used for monitoring the number of incremental scan tasks in TiKV for TiCDC change data capture, showing both ongoing and completed tasks per instance. Essential for diagnosing TiCDC replication lag issues and understanding the workload distribution of incremental data scanning across TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish|ongoing\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 310, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler running commands", "group": "Scheduler", "used_for": "Used for monitoring the number of currently executing scheduler commands on each TiKV instance. Helps identify scheduler bottlenecks, command queue buildup, and potential performance issues when commands accumulate excessively, indicating slow processing or resource contention.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Incremental scan total bytes", "group": "TiKV", "used_for": "Used for monitoring the total bytes scanned by TiCDC's incremental scan operations on each TiKV instance. Essential for tracking data replication volume and diagnosing TiCDC performance issues related to excessive scanning or replication lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 78, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Captured region count", "group": "TiKV", "used_for": "Used for monitoring the number of regions captured by TiCDC on each TiKV instance and their resolution status. Essential for diagnosing TiCDC replication issues, identifying regions that are stuck or slow to replicate, and ensuring change data capture is working properly across all regions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 305, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Non-txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the throttling duration of non-transactional commands (like raw KV operations) in TiKV's flow control mechanism. Helps diagnose when non-txn commands are being rate-limited due to resource pressure, which can indicate storage layer bottlenecks or excessive load from raw KV workloads.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 34, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative volume of data written by TiDB transactions to TiKV. Essential for analyzing transaction write throughput patterns, identifying write-heavy workloads, and diagnosing performance issues related to large transaction sizes or sudden increases in write volume.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Incremental scan speed / s", "group": "TiKV", "used_for": "Used for monitoring the rate of data scanning (bytes per second) performed by TiKV instances during TiCDC's incremental replication process. Essential for diagnosing TiCDC replication performance issues and identifying TiKV instances with slow scan speeds that may cause replication lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 308, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiKV. Helps identify whether high-priority operations (like leader transfers or region splits) are being processed appropriately and diagnose potential priority inversion or scheduling bottlenecks affecting critical cluster operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 195, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Lifetime Reach Counter", "group": "Transaction", "used_for": "Used for monitoring the rate at which transaction TTL (Time To Live) lifetimes are reached in TiDB. Helps identify transactions that are holding locks for too long, which can indicate potential lock contention issues, slow transactions, or applications not properly committing/rolling back transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 146, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV spends seeking old values during CDC (Change Data Capture) operations. Critical for diagnosing CDC replication performance issues and identifying bottlenecks in historical data retrieval for change feeds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 83, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Safepoint OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of successful safepoint load operations per TiDB instance. Helps diagnose transaction GC (garbage collection) coordination issues and identify instances experiencing problems with safepoint synchronization, which is critical for maintaining MVCC consistency across the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ok\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 145, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache size", "group": "TiKV", "used_for": "Used for monitoring TiCDC's old value cache in TiKV, tracking the number of cached entries, total memory usage, and memory quota per instance. Critical for diagnosing TiCDC replication issues and ensuring the old value cache doesn't exceed memory limits, which could impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 312, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler memory quota", "group": "Scheduler", "used_for": "Used for monitoring the scheduler's memory quota usage in TiKV, showing both in-use memory and total capacity per instance. Critical for identifying memory pressure in the scheduler component and preventing out-of-memory issues that could impact transaction processing performance.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 141, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek operation", "group": "TiKV", "used_for": "Used for monitoring the rate of old value scan operations in TiKV's Change Data Capture (CDC) feature, broken down by instance, column family, and tag. Essential for diagnosing CDC performance issues and understanding the I/O overhead of scanning historical data versions during change data replication.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, cf, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 224, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Commit .99 Backoff", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile backoff duration during transaction commits in TiDB. Critical for diagnosing transaction commit delays caused by lock conflicts, region unavailability, or other transient issues that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 313, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn Scheduler Pool Wait Duration", "group": "Scheduler", "used_for": "Used for monitoring the time transactions spend waiting in TiKV's scheduler worker pool queue before being processed. Critical for diagnosing transaction processing bottlenecks and identifying when the scheduler is overloaded, which can cause increased transaction latency and timeouts.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 307, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiKV, including both normal operations and \"too busy\" rejections. Essential for diagnosing transaction processing bottlenecks, identifying when the scheduler is overloaded, and understanding which stages (prewrite, commit, etc.) are experiencing high load or capacity issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 315, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Time used by level", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring time distribution across different priority levels in TiKV's scheduler worker pool. Helps identify if certain priority levels are consuming excessive resources or experiencing bottlenecks, which can indicate scheduling imbalances or workload prioritization issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 321, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Receive Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages received by TiCDC instances from peer nodes. Helps identify communication bottlenecks, network issues, or replication lag by tracking message throughput between TiCDC servers.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 226, "dashboard": "TiDB-Cluster-Clinic", "title": "SafeTS Update Conuter", "group": "Transaction", "used_for": "Used for monitoring the rate of SafeTS (Safe Timestamp) updates between TiDB and TiKV stores, categorized by update result and store. Essential for diagnosing transaction timestamp synchronization issues and identifying stores with failed or slow SafeTS updates that could impact transaction consistency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 311, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the volume of data being written by TiKV's scheduler component on each instance. Helps identify write-heavy instances and diagnose potential write bottlenecks or imbalanced write distribution across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 317, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Running tasks", "group": "Scheduler Worker Pool", "used_for": "**Used For:**\n\nUsed for monitoring the number of currently executing scheduler commands in TiKV's worker pool. Helps identify scheduler bottlenecks, resource saturation, and potential task queuing issues that could impact transaction processing performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the P99, P90, and P80 latency of different transaction execution states in TiDB. Essential for diagnosing transaction performance bottlenecks by identifying which transaction phases (idle, executing, committing, etc.) are experiencing high latency or delays.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 231, "dashboard": "TiDB-Cluster-Clinic", "title": "Assertion", "group": "Transaction", "used_for": "Used for monitoring the rate of prewrite assertion operations in TiDB's transaction processing, categorized by assertion type. Helps diagnose transaction conflicts and optimistic locking issues that can impact transaction latency and throughput.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 143, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache hit", "group": "TiKV", "used_for": "Used for monitoring the cache hit rate of TiCDC's old value cache in TiKV instances. Essential for evaluating TiCDC's change data capture efficiency and identifying memory pressure issues that could impact replication performance when the hit rate drops significantly.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 316, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Scheduler Worker Pool", "used_for": "**Used For:**\n\nUsed for monitoring the probability of scheduler worker threads being at level 0 in TiKV's multilevel feedback queue. Helps diagnose thread scheduling efficiency and identify when workers are spending excessive time at the highest priority level, which may indicate scheduling bottlenecks or workload imbalances in the scheduler worker pool.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 212, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Types Per Second", "group": "Transaction", "used_for": "Used for monitoring the rate of different transaction commit types per second in TiDB, including standard two-phase commit, async commit, and one-phase commit (1PC) transactions. Essential for understanding transaction workload patterns and identifying which commit protocols are being used, helping diagnose transaction performance issues and validate optimization strategies like async commit adoption.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 253, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction with lock execution states duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution (p80, p90, p99) of transaction execution states that hold locks in TiDB. Critical for diagnosing lock contention issues and identifying transactions that may be blocking others, helping pinpoint the root cause of transaction conflicts and deadlocks.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 142, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the latency of TiKV seeking old values during TiCDC change data capture operations, showing p95 and p99 percentiles. Critical for diagnosing TiCDC replication performance issues and identifying when historical value lookups are causing CDC lag or slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 318, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the time tasks spend waiting in the scheduler worker pool queue before execution in TiKV. Critical for identifying scheduler bottlenecks and thread pool saturation that can cause transaction processing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 225, "dashboard": "TiDB-Cluster-Clinic", "title": "Max SafeTS Gap", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the maximum gap between TiDB's SafeTS (safe timestamp) and current time in seconds. Critical for diagnosing stale read issues and transaction consistency problems, as large gaps indicate delays in timestamp synchronization that can affect read performance and data freshness in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 323, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Send Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages sent by TiCDC instances to downstream systems or other peers. Helps identify message throughput bottlenecks, replication lag issues, and abnormal communication patterns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 352, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Batch Size", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message batch sizes in TiCDC's peer-to-peer communication at p50, p95, and p99 percentiles. Helps diagnose message batching efficiency and identify potential network or replication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 319, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Duration of One Time Slice", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the execution duration of individual task time slices in TiKV's scheduler worker pool, tracking p99 and p9999 latencies. Critical for diagnosing scheduler thread pool performance issues, identifying when tasks are taking too long to execute, and detecting potential thread starvation or blocking problems in the storage scheduler.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 321, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Schedule Times", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the execution time distribution of tasks in TiKV's scheduler worker thread pool, tracking p99 and p9999 percentiles. Essential for diagnosing scheduler performance bottlenecks and identifying when transaction scheduling operations experience high latency or tail latency spikes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 354, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive message batch bytes percentiles", "group": "Peer Messages", "used_for": "Used for monitoring the size distribution of message batches received by TiCDC's message server, tracking p50, p95, and p99 percentiles. Essential for diagnosing network efficiency and identifying abnormally large message batches that could indicate replication bottlenecks or memory pressure in peer-to-peer communication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 320, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Execute Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the execution latency of scheduler worker pool tasks in TiKV, tracking p99 and p9999 percentiles. Critical for diagnosing transaction scheduling bottlenecks and identifying when the scheduler worker threads are experiencing high task processing delays that could impact overall transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 356, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive Message Bytes Percentile", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message sizes received by TiCDC's message server at different percentiles (p50, p95, p99). Helps identify abnormally large messages in peer communication and diagnose network bandwidth issues or inefficient data transfer patterns between TiCDC nodes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 261, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction state count change", "group": "Transaction", "used_for": "Used for monitoring the rate of change in transaction state transitions within TiDB sessions. Helps identify transaction state management issues, such as transactions getting stuck in certain states or abnormal state transition patterns that could indicate deadlocks or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 324, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of TiKV scheduler commands at p99 and p9999 percentiles. Essential for diagnosing transaction processing delays and identifying performance bottlenecks in TiKV's transaction scheduler, particularly when investigating slow writes or transaction timeouts.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 255, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the time spent in different transaction execution states and overall transaction duration in TiDB. Helps diagnose transaction performance bottlenecks by identifying which states (e.g., lock waiting, commit, rollback) consume the most time and whether transactions are experiencing delays.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_txn_state_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10023, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo fsync duration", "group": "Redo", "used_for": "Used for monitoring the fsync operation latency in TiCDC's redo log system. Critical for diagnosing redo log write performance issues and identifying disk I/O bottlenecks that could impact data replication reliability and changefeed stability.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 256, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction leave state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions exit different states in TiDB, categorized by transaction type. Helps diagnose transaction lifecycle issues, identify abnormal transaction patterns, and troubleshoot problems related to transaction commit, rollback, or timeout behaviors.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 295, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Usage", "group": "Transaction", "used_for": "Used for monitoring the usage patterns of TiDB's fair locking mechanism in transactions, categorized by lock type. Helps diagnose transaction contention issues and understand how frequently fair locking is being triggered to prevent lock starvation in high-concurrency scenarios.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_session_transaction_fair_locking_usage{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10025, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flush all duration", "group": "Redo", "used_for": "Used for monitoring the time taken to flush all redo log data to storage in TiCDC. Critical for diagnosing redo log write performance issues and identifying bottlenecks in the change data capture pipeline's persistence layer.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flush_all_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 333, "dashboard": "TiDB-Cluster-Clinic", "title": "Pipelined Flush Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of keys flushed in pipelined transaction commits in TiDB. Helps diagnose transaction batching efficiency and identify potential bottlenecks in the pipelined flush mechanism that could impact write throughput.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 358, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Total Stream Count", "group": "Peer Messages", "used_for": "Used for monitoring the total number of active peer message streams in TiCDC and comparing it against the expected stream count based on running TiCDC instances. Helps identify stream connection issues, network problems, or TiCDC node communication failures when actual streams deviate from the expected count (2N-1 for N nodes).", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_message_server_cur_stream_count)", "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}) - 1" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10027, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo write log duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log write operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery feature when redo logs are being persisted to storage.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 296, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Keys", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB's pessimistic transaction model, categorized by locking type. Helps diagnose lock contention issues and identify whether aggressive locking mechanisms are being triggered frequently, which may indicate transaction conflicts or hotspot problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 327, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiKV, tracking p99, p9999, and average values. Essential for identifying write-heavy transactions, detecting abnormal batch sizes, and diagnosing performance issues caused by commands writing excessive keys.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 325, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time transactions spend waiting to acquire latches in TiKV's scheduler, which prevents conflicting transactions from executing concurrently. High wait durations indicate transaction contention on the same keys, helping diagnose write conflicts and hotspot issues that impact transaction throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10033, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write bytes/s ", "group": "Redo", "used_for": "Used for monitoring the redo log write throughput in bytes per second for TiCDC changefeeds. Essential for tracking redo log I/O performance and identifying potential bottlenecks in the redo logging system that could impact data replication reliability.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 323, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of scheduler stage operations and \"too busy\" rejections in TiKV's transaction scheduler for specific command types. Essential for diagnosing transaction processing bottlenecks, identifying when TiKV is overloaded and rejecting requests, and understanding which scheduler stages are experiencing high load or contention.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10029, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flush log duration", "group": "Redo", "used_for": "Used for monitoring the time taken to flush redo logs to storage in TiCDC. Critical for diagnosing replication lag issues and identifying bottlenecks in the redo log persistence layer that could impact data consistency during disaster recovery scenarios.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 154, "dashboard": "TiDB-Cluster-Clinic", "title": "Compile Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL compilation latency in TiDB, broken down by SQL statement type. Essential for diagnosing slow query plan generation issues and identifying when the optimizer is taking excessive time to compile SQL statements before execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 199, "dashboard": "TiDB-Cluster-Clinic", "title": "Pessimistic Statement Retry OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of pessimistic transaction statement retries in TiDB. Helps identify transaction conflicts and contention issues that cause statements to retry, which can indicate lock conflicts or write-write conflicts in pessimistic transaction mode.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 334, "dashboard": "TiDB-Cluster-Clinic", "title": "Pipelined Flush Size", "group": "Transaction", "used_for": "Used for monitoring the distribution of batch sizes when TiDB flushes pipelined transaction writes to TiKV. Helps diagnose transaction batching efficiency and identify whether writes are being properly aggregated before sending to storage, which impacts write throughput and network efficiency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 156, "dashboard": "TiDB-Cluster-Clinic", "title": "Parse Duration", "group": "Executor", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile SQL parsing latency in TiDB, broken down by SQL statement type. Essential for identifying parsing bottlenecks when queries take unexpectedly long to execute, particularly with complex SQL statements or when experiencing high CPU usage during the parse phase.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 335, "dashboard": "TiDB-Cluster-Clinic", "title": "Pipelined Flush Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of pipelined flush operations during transaction commits in TiDB, showing p95, p99, and p999 percentiles. Critical for diagnosing transaction commit performance issues and identifying when the pipelined memory buffer flush to TiKV is experiencing delays or bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_pipelined_flush_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/ sum(rate(tidb_tikvclient_pipelined_flush_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 350, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Stream Count Between Nodes", "group": "Peer Messages", "used_for": "**Used For:**\n\nUsed for monitoring the current number of active message streams between TiCDC nodes in the cluster. Helps diagnose inter-node communication issues, connection stability problems, and potential network bottlenecks affecting data replication between TiCDC instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "ticdc_message_server_cur_stream_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 328, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the distribution and characteristics of key-value scan operations in TiKV's scheduler for specific commands. Helps diagnose scan performance issues by breaking down scan details by tags, useful for identifying inefficient range queries or excessive data scanning patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 331, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the detailed scan operations in TiKV's scheduler for the default column family, broken down by operation tags. Helps diagnose performance issues related to key-value scanning patterns, such as excessive range scans or inefficient query access patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10031, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write rows/s ", "group": "Redo", "used_for": "Used for monitoring the rate of rows written to TiCDC's redo log per second, broken down by instance/type and changefeed. Essential for tracking redo log write throughput and identifying changefeeds with high data volume or potential write bottlenecks in the disaster recovery pipeline.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance,type)", "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[2m])) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 169, "dashboard": "TiDB-Cluster-Clinic", "title": "Execution Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile execution duration of SQL statements in TiDB, broken down by SQL type (SELECT, INSERT, UPDATE, etc.). Essential for identifying slow query execution patterns and diagnosing performance degradation in specific SQL operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 329, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations specifically on the lock column family during scheduler command execution in TiKV. Essential for diagnosing lock-related performance issues, identifying excessive lock scanning, and troubleshooting transaction conflicts or deadlock scenarios.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 332, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command read duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the read operation latency in TiKV's scheduler component, tracking both p99 and p9999 percentiles. Essential for diagnosing slow read requests and identifying tail latency issues in the transaction scheduler's read processing pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimated Live Objects", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring the number of live objects in TiCDC's Go runtime heap memory. Helps diagnose memory leaks and excessive object allocation issues that could lead to increased garbage collection pressure and performance degradation.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10035, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Redo", "used_for": "Used for monitoring the busy ratio percentage of TiCDC redo log workers, indicating how much time workers spend actively processing versus idle. Essential for identifying redo log processing bottlenecks and determining if workers are saturated, which could lead to replication lag or require scaling adjustments.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])*100) by (changefeed,instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 330, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the detailed scan operations on TiKV's write column family during scheduler command execution. Helps diagnose write-heavy workload performance by tracking scan patterns, identifying inefficient range queries, and detecting potential issues with write CF compaction or data distribution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 157, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Usage", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory consumption, including resident memory, heap allocation, and GC thresholds. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC's memory configuration during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 76, "dashboard": "TiDB-Cluster-Clinic", "title": "Expensive Executors OPS", "group": "Executor", "used_for": "Used for monitoring the rate of expensive executor operations in TiDB, categorized by executor type. Helps identify query patterns that consume excessive resources (memory/CPU), enabling quick detection of inefficient queries that may impact cluster performance or cause OOM issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 335, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the distribution of coprocessor request latencies in TiKV. Essential for diagnosing slow coprocessor operations that handle table scans, aggregations, and other pushed-down computations from TiDB, helping identify performance bottlenecks in analytical queries.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 249, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Table Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of read operations served directly from TiDB's table cache feature. Helps assess table cache effectiveness and identify whether cached small tables are successfully reducing storage layer access for frequently queried data.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_read_from_tablecache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 336, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring coprocessor request latency in TiKV, tracking both P99 and P9999 percentiles to identify tail latency issues. Essential for diagnosing slow analytical queries and table scan performance problems, as coprocessor handles pushed-down computation from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 269, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Memory Usage", "group": "Executor", "used_for": "Used for monitoring the memory consumption of TiDB's plan cache on each instance. Essential for diagnosing memory pressure issues and determining if plan cache memory usage is contributing to OOM problems or needs tuning.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 160, "dashboard": "Prometheus-TiCDC-Clinic", "title": "GC STW Duration (last 256 GC cycles)", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring Go garbage collection stop-the-world (STW) pause durations in TiCDC instances over the last 256 GC cycles. Critical for diagnosing TiCDC replication lag or throughput issues caused by excessive GC pauses that block normal processing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 258, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Miss OPS", "group": "Executor", "used_for": "Used for monitoring the rate of plan cache misses per second in TiDB, categorized by miss type. Essential for diagnosing SQL execution performance issues caused by inefficient plan cache utilization, which can lead to increased query compilation overhead and reduced throughput.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10002, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication of DDL schema changes to Pulsar topics in TiCDC. Essential for verifying that schema evolution events are properly propagated to downstream Pulsar consumers and diagnosing DDL replication failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 161, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Allocator Throughput", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory allocation throughput, including allocation rate, deallocation rate, and object allocation frequency. Essential for diagnosing memory pressure issues, identifying memory leaks, and understanding garbage collection behavior in TiCDC instances.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 271, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Plan Num", "group": "Executor", "used_for": "Used for monitoring the total number of cached execution plans in TiDB's plan cache per instance. Helps diagnose plan cache effectiveness and memory usage, particularly when investigating query performance issues or determining if the plan cache size needs adjustment.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_plan_num_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 91, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache feature, broken down by cache type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 337, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Requests", "group": "Coprocessor Overview", "used_for": "Used for monitoring the total rate of coprocessor requests processed by TiKV instances, broken down by request type. Essential for tracking coprocessor workload patterns and identifying unusual spikes in analytical query activity or table scan operations.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 338, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Request Errors", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV, categorized by error reason. Essential for diagnosing query execution failures, identifying problematic coprocessor operations, and troubleshooting issues like region errors, timeout errors, or resource constraints that prevent successful query processing.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 311, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Counter", "group": "Executor", "used_for": "Used for monitoring the rate of MPP (Massively Parallel Processing) coordinator operations in TiDB, broken down by operation type. Essential for diagnosing MPP query execution issues and understanding the coordinator's workload when troubleshooting distributed analytical query performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_mpp_coordinator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10001, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of DDL schema changes published to Pulsar topics by TiCDC. Essential for tracking schema synchronization progress and diagnosing issues when DDL operations (like CREATE/ALTER TABLE) are not being replicated to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 339, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of key scanning operations performed by TiKV's coprocessor during query execution. Essential for diagnosing coprocessor performance issues and identifying queries that scan excessive amounts of data, which can indicate missing indexes or inefficient query patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10003, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring DDL schema publication failures when TiCDC replicates data to Pulsar. Critical for diagnosing issues where DDL changes fail to be published to Pulsar topics, which could cause data inconsistency or replication interruptions in downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10004, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the rate of DML schema changes published to Pulsar topics by TiCDC. Helps identify schema evolution activity and troubleshoot issues with schema synchronization to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 326, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiKV, tracking P99, P9999, and average values. Essential for diagnosing I/O performance issues and identifying commands that scan excessive keys, which may indicate inefficient queries or hotspot problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 340, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the number of keys scanned per coprocessor request in TiKV, tracking p99 and p9999 percentiles. Essential for diagnosing inefficient queries with excessive key scans, identifying full table scans, and detecting coprocessor performance bottlenecks caused by scanning too many keys.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 312, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Latency", "group": "Executor", "used_for": "Used for monitoring the latency of MPP (Massively Parallel Processing) coordinator operations in TiDB at different percentiles (p50, p75, p95). Essential for diagnosing performance issues in distributed analytical queries that use TiFlash, helping identify coordination overhead and bottlenecks in MPP query execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 75, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.75, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10005, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication rate of DML schema changes to Pulsar topics in TiCDC. Helps verify that schema changes are being correctly propagated to downstream Pulsar systems and identify potential schema synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 341, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total RocksDB Perf Statistics", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of internal delete operations skipped by RocksDB during coprocessor requests in TiKV. Helps diagnose storage engine efficiency and identify potential issues with MVCC garbage collection or compaction that could impact coprocessor query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql QPS", "group": "Distsql", "used_for": "Used for monitoring the distributed SQL query throughput in TiDB, broken down by coprocessor request type (table scan, index scan, etc.). Essential for identifying query load patterns and diagnosing performance issues related to distributed query execution between TiDB and TiKV.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (copr_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 342, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Response Size", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of data returned by TiKV coprocessor requests per instance. Helps identify which TiKV nodes are handling heavy read workloads and diagnose potential network bandwidth bottlenecks or unbalanced data access patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Partial QPS", "group": "Distsql", "used_for": "Used for monitoring the rate of partial distributed SQL scan operations per second in TiDB. Helps identify when queries are performing partial scans (scanning subset of keys), which is useful for diagnosing query execution patterns and potential performance issues related to distributed query processing.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10007, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client bytes published ", "group": "Pulsar Sink", "used_for": "Used for monitoring the total bytes published by TiCDC's Pulsar client to the Pulsar message queue, broken down by changefeed and instance. Essential for tracking data replication throughput to Pulsar and diagnosing network bandwidth issues or replication bottlenecks in the Pulsar sink pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10006, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring failed DML schema publications to Pulsar topics in TiCDC. Critical for identifying schema synchronization issues when replicating data changes to Pulsar, helping diagnose why certain table schemas fail to publish and may cause replication delays or data inconsistencies.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 343, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Quota", "group": "Coprocessor Overview", "used_for": "Used for monitoring the memory quota allocated to TiKV's coprocessor component across instances. Helps identify memory allocation patterns and potential memory pressure issues when coprocessor requests consume excessive resources, which can lead to OOM errors or query failures.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10009, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client rpc count", "group": "Pulsar Sink", "used_for": "Used for monitoring the total number of RPC calls made by TiCDC's Pulsar client to the Pulsar cluster, broken down by changefeed and instance. Helps diagnose communication issues between TiCDC and Pulsar, identify abnormal RPC patterns, and troubleshoot data replication problems in Pulsar sink scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 345, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the latency of coprocessor request handling in TiKV, tracking p99 and p9999 percentiles. Essential for diagnosing slow coprocessor operations (table scans, aggregations, index lookups) and identifying tail latency issues that impact query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 57, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per DistSQL query at different percentiles (p50, p90, max). Essential for identifying inefficient queries with excessive key scans, which can indicate missing indexes, full table scans, or poorly optimized query patterns causing performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10011, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer rpc latency", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile RPC latency of Pulsar client producers in TiCDC's Pulsar sink. Critical for identifying tail latency issues when TiCDC replicates data to Pulsar, helping diagnose slow replication and potential bottlenecks in the Pulsar message publishing pipeline.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10008, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client connections opened", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of active Pulsar client connections opened by TiCDC changefeeds. Helps diagnose connection management issues, detect connection leaks, and verify proper connectivity between TiCDC and Pulsar message brokers during data replication.", "component": "TICDC", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 347, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time coprocessor requests spend waiting in TiKV's queue before execution, tracking p99 and p9999 percentiles. Critical for diagnosing coprocessor bottlenecks and thread pool saturation when queries experience delays before processing begins.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 349, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Requests", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) coprocessor requests processed by TiKV instances, broken down by vectorization type. Essential for tracking coprocessor workload intensity and identifying whether queries are using vectorized or non-vectorized execution, which helps diagnose query performance and resource utilization issues.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10010, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer latency ", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC's Pulsar client producer when writing change data to Pulsar message queues. Critical for diagnosing replication delays and identifying performance bottlenecks in the TiCDC-to-Pulsar data pipeline, especially when changefeeds experience slow message publishing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r", "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 175, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Cache", "group": "Distsql", "used_for": "Used for monitoring the coprocessor cache hit/miss rates in TiDB's distributed SQL execution. Helps diagnose query performance issues by showing how effectively the coprocessor cache is reducing redundant requests to TiKV, with higher hit rates indicating better cache utilization and improved query efficiency.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 333, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check memory locks duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time spent checking memory locks during transaction processing in TiKV's scheduler. Critical for diagnosing transaction conflicts and lock contention issues that can cause increased latency in write-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of keys scanned in partial results during distributed SQL execution at p80, p90, and p100 percentiles. Helps diagnose query efficiency issues by identifying queries that scan excessive keys, which can indicate missing indexes, inefficient query plans, or table scan problems affecting query performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Seconds 999", "group": "Distsql", "used_for": "Used for monitoring the 99.9th percentile latency of coprocessor requests from TiDB to TiKV. Critical for identifying tail latency issues in distributed query execution, particularly when diagnosing slow analytical queries or table scans that push computation down to TiKV.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{store!=\"0\", scope=\"false\", type=\"Cop\", k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 346, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring coprocessor request handling latency across individual TiKV stores, showing P99 and P99.99 percentiles plus average duration. Critical for identifying which specific TiKV instances have slow coprocessor performance, helping diagnose uneven load distribution or storage-level bottlenecks affecting analytical queries.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 353, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations across different column families in TiKV's coprocessor. Helps diagnose coprocessor throughput issues and identify which column families are experiencing high scan activity during query execution.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10012, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in TiCDC's Pulsar producer queue that are waiting to be sent to Pulsar. Critical for identifying message backlog issues, producer bottlenecks, or downstream Pulsar cluster performance problems that could lead to replication lag.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10013, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in the Pulsar client producer queue for TiCDC changefeeds. Helps identify message backlog issues and potential bottlenecks in data replication to Pulsar, indicating when the producer cannot keep up with the change data capture rate.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Duration", "group": "Distsql", "used_for": "Used for monitoring the latency distribution of distributed SQL (DistSQL) query execution in TiDB at various percentiles (p50, p90, p99, p999). Essential for diagnosing performance issues when TiDB pushes query operations down to TiKV, helping identify slow distributed query processing and coprocessor request bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 352, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations performed by TiKV's coprocessor across the cluster. Essential for analyzing index query throughput and identifying when excessive index scans may be causing performance bottlenecks or resource contention.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 348, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time coprocessor requests spend waiting in queue before execution across different TiKV stores, tracking p99 and p9999 percentiles. Critical for diagnosing coprocessor request queuing issues and identifying stores experiencing thread pool saturation or resource contention that causes elevated wait times.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 350, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Executors", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) executor operations in TiKV's coprocessor, broken down by executor type. Helps identify which coprocessor operations (like TableScan, Selection, Aggregation) are most frequently used and diagnose performance bottlenecks in pushdown query execution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "TiDB-Cluster-Clinic", "title": "TiClient Region Error OPS", "group": "KV Errors", "used_for": "**Used For:**\n\nUsed for monitoring the rate of region-level errors encountered by TiDB when accessing TiKV, broken down by error type (including server_is_busy). Critical for diagnosing region availability issues, hotspot problems, and TiKV overload conditions that impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 53, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of KV client backoff operations in TiDB, categorized by backoff type. Essential for identifying retry patterns and diagnosing issues like region unavailability, lock conflicts, or TiKV overload that force TiDB to back off and retry requests.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 354, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations across different column families in TiKV's RocksDB storage engine. Helps diagnose index scan throughput and identify which column families are handling the most index read operations during query execution.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff Duration", "group": "KV Errors", "used_for": "Used for monitoring the duration of KV client backoff operations at different percentiles (p80, p99, p999) when TiDB encounters errors communicating with TiKV. Essential for diagnosing performance degradation caused by region unavailability, lock conflicts, or other transient TiKV errors that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 172, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request OPS", "group": "KV Request", "used_for": "Used for monitoring the operations per second (OPS) of different KV request types sent from TiDB to TiKV, broken down by TiDB instance. Essential for identifying which TiDB nodes are generating high request volumes and diagnosing uneven load distribution or sudden traffic spikes in the KV layer.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 48, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by store", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of KV requests from TiDB to individual TiKV stores. Essential for identifying performance bottlenecks and uneven load distribution across TiKV nodes, helping diagnose which specific stores are experiencing slow request processing.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le, store))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 32, "dashboard": "TiDB-Cluster-Clinic", "title": "Lock Resolve OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of lock resolution operations in TiDB's transaction processing system. Essential for diagnosing transaction conflicts and identifying when the cluster is experiencing high contention due to locks that need to be resolved or cleaned up.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 355, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time spent checking memory locks during coprocessor operations in TiKV. Helps diagnose performance bottlenecks when coprocessor requests experience delays due to memory lock contention or validation overhead.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 351, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations in TiKV's coprocessor, broken down by scan type (select queries). Essential for identifying excessive full table scans that may indicate missing indexes or inefficient query patterns causing performance degradation.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 356, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the latency of memory lock checks during coprocessor operations in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing performance bottlenecks in pessimistic transaction processing, especially when concurrent transactions are contending for the same data in memory.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 359, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of tasks being scheduled at level 0 in TiKV's unified read pool. Helps diagnose read pool scheduling efficiency and identify potential bottlenecks when too many tasks are queued at the lowest priority level, which may indicate resource contention or suboptimal workload distribution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 223, "dashboard": "TiDB-Cluster-Clinic", "title": "Replica Selector Failure Per Second", "group": "KV Errors", "used_for": "Used for monitoring the rate of replica selector failures per second in TiDB's TiKV client, categorized by failure type. Critical for diagnosing issues with TiDB's ability to select appropriate TiKV replicas for read/write operations, which can indicate network problems, region unavailability, or load balancing issues.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 30, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by type", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of different KV request types (Get, Scan, Prewrite, Commit, etc.) sent from TiDB to TiKV. Essential for identifying which specific request types are experiencing high latency and diagnosing TiDB-TiKV communication bottlenecks or slow transaction operations.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le,type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 361, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the time tasks spend waiting in the queue before being executed by TiKV's unified read pool. Critical for diagnosing read request queueing issues and identifying when the read pool is saturated or experiencing scheduling delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 250, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache Error OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of region cache errors in TiDB's TiKV client, categorized by error type. Essential for diagnosing region routing issues, stale region metadata problems, and network connectivity failures that can cause query retries and performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"err\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 360, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiKV's unified read pool, broken down by priority level. Essential for diagnosing read request congestion, identifying priority-based resource contention, and detecting when the read pool is saturated or unevenly loaded.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 358, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring time distribution across different priority levels in TiKV's unified read pool. Helps identify if certain priority levels are consuming excessive resources or experiencing bottlenecks, which is critical for diagnosing read request scheduling issues and ensuring fair resource allocation among different workload priorities.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 219, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded between TiKV stores in TiDB. Helps identify request routing patterns and diagnose issues when requests need to be redirected from one store to another, which may indicate leader transfers, region migrations, or store failures.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (from_store, to_store, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 362, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Duration of One Time Slice", "group": "Unified Read Pool", "used_for": "Used for monitoring the execution duration of individual task polling cycles in TiKV's unified read pool thread workers. Critical for diagnosing read performance issues and identifying whether tasks are experiencing excessive processing time or thread pool contention, with p99 and p9999 percentiles helping detect tail latency problems in read operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "TiDB-Cluster-Clinic", "title": "Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of partial results generated during distributed SQL execution in TiDB, showing p50, p90, and maximum values. Helps diagnose query execution efficiency and identify queries that require excessive data shuffling or produce fragmented results across multiple TiKV regions.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 364, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Schedule Times", "group": "Unified Read Pool", "used_for": "Used for monitoring task execution latency in TiKV's unified read pool, tracking p99 and p9999 percentiles. Essential for diagnosing read performance bottlenecks and identifying whether slow queries are caused by task scheduling delays in the thread pool.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 306, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Hit/Miss OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests that hit or miss the local replica in TiDB. Helps evaluate the effectiveness of stale read optimization and diagnose issues when stale reads frequently miss local data, requiring fallback to leader reads.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 367, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks duration", "group": "GC", "used_for": "Used for monitoring the latency of garbage collection tasks in TiKV, tracking both p99 and p9999 percentiles. Essential for diagnosing GC performance issues that could impact cluster stability, especially when GC tasks take too long and cause memory pressure or affect transaction processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 366, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection tasks in TiKV, including successful executions, skipped tasks, and failures. Essential for diagnosing GC performance issues, identifying task bottlenecks, and detecting abnormal GC behavior that could impact storage space reclamation and cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 363, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Execute Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the execution duration of read tasks in TiKV's unified read pool, tracking p99 and p9999 latencies. Essential for diagnosing read performance bottlenecks and identifying whether slow queries are caused by task execution delays in the storage layer.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 263, "dashboard": "TiDB-Cluster-Clinic", "title": "RPC Layer Latency", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile network latency between TiDB and TiKV stores at the RPC layer. Critical for diagnosing network-related performance issues, identifying slow TiKV nodes, and distinguishing between storage engine delays versus network transmission delays in KV request processing.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store))", "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 220, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS by Type", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded by TiDB to other TiKV nodes, broken down by request type and result status. Essential for diagnosing follower read performance, identifying forwarding bottlenecks, and understanding request routing patterns when TiDB cannot serve requests from local replicas.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 262, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Region Duration", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of loading region information into TiDB's region cache from PD. Critical for diagnosing slow KV request issues caused by region metadata lookup delays, especially when experiencing sudden traffic pattern changes or region splits.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 254, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction enter state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions enter different states (e.g., idle, running, committed) in TiDB. Helps diagnose transaction lifecycle issues, identify abnormal state transitions, and understand transaction processing patterns during performance troubleshooting.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 307, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests per second in TiDB, broken down by request type. Essential for tracking the usage and throughput of stale read operations, which allow reading slightly outdated data to reduce latency and improve read scalability.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_req_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 308, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req Traffic", "group": "KV Request", "used_for": "Used for monitoring the network traffic volume of stale read requests in TiDB, broken down by result status and direction (sent/received). Essential for tracking the I/O overhead of stale read operations and diagnosing performance issues when using follower read or historical read features.", "component": "TIDB", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, direction)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 164, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache OK OPS", "group": "KV Request", "used_for": "Used for monitoring the successful operations per second on TiDB's region cache, broken down by operation type. Helps assess region cache efficiency and identify when the cache is effectively serving region location requests without errors.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 371, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC Progress", "group": "GC", "used_for": "Used for monitoring the progress of TiKV's automatic garbage collection by tracking the ratio of processed regions to total regions. Helps identify GC bottlenecks and ensures garbage collection is completing efficiently across all TiKV instances.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 368, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC seconds", "group": "GC", "used_for": "Used for monitoring the latency of garbage collection (GC) operations initiated by TiDB to clean up obsolete MVCC versions in TiKV. Essential for diagnosing GC performance issues that could lead to storage bloat, increased read latency, or cluster instability when GC cannot keep up with write workload.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 370, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "ResolveLocks Progress", "group": "GC", "used_for": "Used for monitoring the progress and results of lock resolution operations during garbage collection in TiKV. Essential for diagnosing GC efficiency issues and identifying problems with cleaning up stale transaction locks that could block new transactions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 337, "dashboard": "TiDB-Cluster-Clinic", "title": "Client-side slow score", "group": "KV Request", "used_for": "Used for monitoring the slow score of each TiKV store as perceived by TiDB clients. Helps identify problematic TiKV stores experiencing performance degradation or network issues that may cause request slowdowns, enabling quick isolation of unhealthy nodes.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_store_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 338, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV-side slow score", "group": "KV Request", "used_for": "Used for monitoring the slow score feedback from each TiKV store as perceived by TiDB clients. Helps identify which TiKV stores are experiencing performance degradation or responding slowly, enabling quick diagnosis of unbalanced load or problematic nodes in the cluster.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_feedback_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 159, "dashboard": "TiDB-Cluster-Clinic", "title": "Async TSO Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of asynchronous TSO (Timestamp Oracle) requests from TiDB to PD at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication bottlenecks that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 77, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO Wait Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the wait duration (p90, p99, p999) when TiDB requests TSO (Timestamp Oracle) from PD. Critical for diagnosing transaction latency issues caused by PD client bottlenecks or network delays between TiDB and PD servers.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 377, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan write details", "group": "GC", "used_for": "Used for monitoring the rate of keys scanned during garbage collection operations in TiKV's write column family, broken down by key mode and tag. Essential for diagnosing GC performance issues, understanding GC workload patterns, and identifying potential bottlenecks in the write CF cleanup process.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 372, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC speed", "group": "GC", "used_for": "Used for monitoring the rate at which TiKV's garbage collection (GC) process deletes obsolete MVCC versions. Essential for assessing GC efficiency and diagnosing issues when deleted data isn't being cleaned up quickly enough, which can lead to storage bloat and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of PD client commands (excluding TSO operations) from TiDB instances at p90, p99, and p999 percentiles. Essential for diagnosing communication delays between TiDB and PD, helping identify network issues, PD overload, or metadata operation bottlenecks that impact query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 78, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO RPC Duration", "group": "PD Client", "used_for": "Used for monitoring the tail latency of TSO (Timestamp Oracle) RPC requests from TiDB to PD at p99, p999, and p9999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication issues that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99, 999, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 79, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO OPS", "group": "PD Client", "used_for": "Used for monitoring the throughput of TSO (Timestamp Oracle) requests from TiDB to PD, measured in operations per second. Essential for identifying TSO request volume patterns and detecting potential bottlenecks in timestamp allocation, which is critical for transaction processing performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))", "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 375, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC interval", "group": "GC", "used_for": "Used for monitoring the configured garbage collection (GC) interval in TiKV, which determines how frequently GC operations run to clean up old MVCC versions. Essential for understanding GC scheduling behavior and diagnosing issues related to storage space accumulation or GC frequency tuning.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 369, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC worker actions", "group": "GC", "used_for": "Used for monitoring the rate of different garbage collection (GC) worker actions in TiDB, broken down by action type. Essential for diagnosing GC performance issues, identifying bottlenecks in old version cleanup, and ensuring the GC process is running smoothly to reclaim storage space.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed PD client commands from TiDB instances, categorized by command type. Essential for identifying communication issues between TiDB and PD, helping diagnose cluster metadata access problems, TSO allocation failures, or network connectivity issues that could impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 20, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second (OPS) of different PD client commands from TiDB instances, excluding TSO requests. Helps identify which PD API calls (like GetRegion, GetStore) are most frequently used and diagnose potential bottlenecks in TiDB's communication with PD.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"tso\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 376, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC in Compaction Filter", "group": "GC", "used_for": "Used for monitoring the effectiveness of TiKV's garbage collection during RocksDB compaction operations. Tracks the rate of filtered obsolete versions, skipped keys, and MVCC rollback operations, helping diagnose GC efficiency issues and identify whether compaction filter is properly cleaning up stale data to prevent storage bloat.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Pipeline Duration", "group": "Duration", "used_for": "Used for monitoring the 99th percentile latency of TiKV's write pipeline stages, including Raft log append duration, request wait time, and apply wait time. Critical for diagnosing write performance bottlenecks and identifying which stage of the write path is causing slowdowns during write-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 381, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Handled tasks", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of different task types handled by TiKV's lock manager in pessimistic locking scenarios. Essential for diagnosing lock contention issues, deadlock detection performance, and understanding the distribution of lock-related operations across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 380, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Thread CPU", "group": "Pessimistic Locking", "used_for": "Used for monitoring CPU usage of TiKV's lock manager threads, including waiter manager and deadlock detector threads. Critical for diagnosing performance issues in pessimistic locking scenarios, especially when experiencing lock contention or deadlock detection overhead.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store size", "group": "Cluster", "used_for": "Used for monitoring the actual disk space consumed by each TiKV store instance. Essential for tracking storage capacity utilization, planning capacity expansion, and identifying stores approaching disk space limits before they impact cluster availability.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Available size", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the available disk space on each TiKV instance. Critical for preventing storage exhaustion issues and ensuring sufficient capacity for data writes and compaction operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572011, "dashboard": "TiDB-Cluster-Clinic", "title": "Estimate TSO RTT Latency", "group": "PD Client", "used_for": "Used for monitoring the estimated round-trip time (RTT) latency between TiDB instances and PD when requesting TSO (Timestamp Oracle) allocations. Critical for diagnosing timestamp allocation delays that can impact transaction performance and identifying network or PD service issues affecting distributed transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_client_request_estimate_tso_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 378, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan default details", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection key scanning operations in TiKV's default column family, broken down by key mode and tag. Helps diagnose GC performance issues and understand the volume of keys being processed during garbage collection cycles.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 373, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC SafePoint", "group": "GC", "used_for": "Used for monitoring the Auto GC (Garbage Collection) safe point timestamp in TiKV, which indicates the oldest data version that must be retained. Essential for diagnosing GC progress issues, MVCC version accumulation problems, and ensuring that historical data cleanup is functioning properly to prevent storage bloat.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 222, "dashboard": "TiDB-Cluster-Clinic", "title": "Request forwarded status", "group": "PD Client", "used_for": "Used for monitoring whether TiDB's PD client requests are being forwarded to other PD nodes instead of being handled directly by the connected PD leader. Helps diagnose PD leadership changes, network routing issues, or PD cluster instability that causes request redirection overhead.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "pd_client_request_forwarded_status" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 247, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of HTTP API requests from TiDB to PD (Placement Driver). Helps identify communication patterns between TiDB and PD, diagnose connectivity issues, and detect abnormal request spikes that may indicate metadata access problems or cluster coordination issues.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 374, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC lifetime", "group": "GC", "used_for": "Used for monitoring the configured GC (Garbage Collection) lifetime setting in TiKV, which determines how long historical MVCC versions are retained before cleanup. Essential for diagnosing issues related to storage space growth, snapshot read failures, or balancing between historical data access requirements and disk usage optimization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 383, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Waiting Queue", "group": "Pessimistic Locking", "used_for": "Used for monitoring the size and status of pessimistic lock waiting queues in TiKV. Critical for diagnosing lock contention issues, identifying transactions blocked by lock waits, and detecting potential deadlock scenarios that impact transaction throughput.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) ", "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Capacity size", "group": "Cluster", "used_for": "Used for monitoring the total storage capacity of each TiKV instance in the cluster. Essential for capacity planning and identifying instances approaching storage limits before they run out of disk space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 246, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of HTTP requests from TiDB to PD (Placement Driver) at different percentiles (p90, p99, p999). Essential for diagnosing communication issues between TiDB and PD, identifying network bottlenecks, or PD performance problems that could impact metadata operations and transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 386, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detector leader", "group": "Pessimistic Locking", "used_for": "Used for identifying which TiKV instance is currently serving as the deadlock detector leader in pessimistic locking scenarios. Essential for troubleshooting deadlock detection issues and verifying proper leader election when investigating transaction conflicts or lock-related problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cop Read Duration", "group": "Duration", "used_for": "Used for monitoring the end-to-end latency of coprocessor read requests in TiKV at p95 and p99 percentiles, including snapshot acquisition time, queue wait time, and request handling time. Critical for diagnosing slow table scans, index lookups, and identifying whether performance bottlenecks are caused by storage engine delays, request queuing, or processing overhead.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.95,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 248, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed HTTP requests from TiDB to PD (Placement Driver). Essential for identifying communication issues between TiDB and PD, such as network problems, PD unavailability, or API errors that could impact cluster metadata operations and query routing.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 385, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Detect error", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of errors occurring in TiKV's pessimistic lock manager, categorized by error type. Critical for diagnosing deadlock detection failures, lock timeout issues, and other pessimistic transaction locking problems that could impact transaction success rates.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 382, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Waiter lifetime duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring how long transactions wait to acquire pessimistic locks in TiKV's lock manager. Critical for diagnosing lock contention issues and identifying transactions experiencing excessive wait times due to conflicting locks, with p99 and p9999 percentiles helping detect tail latency in lock acquisition.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 384, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detect duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of deadlock detection operations in TiKV's pessimistic locking system, showing p99 and p9999 percentiles. Critical for diagnosing performance issues when transactions experience delays due to deadlock detection overhead, especially during high contention scenarios with concurrent pessimistic transactions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 387, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total pessimistic locks memory size", "group": "Pessimistic Locking", "used_for": "Used for monitoring the total memory consumed by pessimistic locks in TiKV instances. Critical for diagnosing memory pressure issues and identifying potential lock accumulation problems that could lead to out-of-memory errors or transaction conflicts.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 315, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB Wait TSO Future Duration", "group": "PD Client", "used_for": "Used for monitoring the duration TiDB waits when a TSO (timestamp oracle) timestamp is in the future, tracking p90, p99, and p999 latencies. Critical for diagnosing clock synchronization issues between TiDB and PD servers, which can cause transaction delays and performance degradation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_ts_future_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le) / sum(rate(tidb_tikvclient_ts_future_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of TiKV instances, showing actual CPU usage rate against the allocated CPU quota. Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiKV nodes need vertical scaling or load rebalancing.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",component=~\".*tikv\"}\n [$__rate_interval]\n)) by (instance) ", "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",component=~\".*tikv\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing slow DDL execution, schema synchronization delays, and identifying instances experiencing performance issues when fetching or updating table metadata from TiKV.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring TiKV instance memory usage and comparing it against configured memory quotas. Essential for detecting memory pressure, potential OOM risks, and verifying that TiKV instances are operating within their allocated memory limits.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",component=~\".*tikv\"}\n \n)) by (instance) ", "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",component=~\".*tikv\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 388, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "In-memory pessimistic locking result", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate and outcome of in-memory pessimistic lock operations in TiKV. Helps diagnose lock contention issues and identify whether pessimistic locks are succeeding or failing, which is critical for troubleshooting transaction conflicts and performance degradation in pessimistic transaction mode.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring disk I/O utilization percentage across TiKV instances in the cluster. Essential for identifying storage bottlenecks, detecting overloaded disks, and diagnosing performance degradation caused by I/O saturation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 251, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Action Duration", "group": "Schema Load", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency of schema loading operations in TiDB, broken down by action type (e.g., reload, diff). Critical for diagnosing DDL performance issues and identifying slow schema synchronization that can impact query execution and cluster stability.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, action))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 29, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Lease Error OPM", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema lease errors per minute in TiDB instances. Critical for detecting schema synchronization issues between TiDB servers, which can cause query failures or inconsistent metadata when the schema lease mechanism fails to maintain consistency across the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 28, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema loading operations in TiDB instances, categorized by operation type. Essential for diagnosing schema synchronization issues, identifying excessive schema reloads, and troubleshooting DDL-related performance problems across the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 13, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errps", "group": "Cluster", "used_for": "Used for monitoring error rates across TiKV instances, including gRPC message failures, PD heartbeat anomalies, and critical errors. Essential for quickly identifying unhealthy TiKV nodes and diagnosing cluster stability issues that could impact data availability and query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "MBps", "group": "Cluster", "used_for": "Used for monitoring the throughput (megabytes per second) of TiKV storage engine I/O operations, including WAL writes, data reads, and in-memory engine operations across cluster instances. Essential for identifying storage bottlenecks, unbalanced I/O distribution, and diagnosing performance issues related to disk bandwidth saturation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 157, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Data From Cached Table Duration", "group": "Schema Load", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency when loading data from TiDB's cached tables feature. Critical for diagnosing performance issues with table cache operations and identifying instances where cache loading becomes a bottleneck during query execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 389, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pessimistic lock activities", "group": "Pessimistic Locking", "used_for": "Used for monitoring the number of transactions waiting in pessimistic lock queues, categorized by lock type. Essential for diagnosing lock contention issues, identifying blocking scenarios, and understanding transaction conflicts in pessimistic transaction mode.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 390, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lengths of lock wait queues when transaction enqueues", "group": "Pessimistic Locking", "used_for": "Used for monitoring the distribution of lock wait queue lengths when transactions enqueue for pessimistic locks in TiKV. Helps diagnose lock contention issues by showing how many transactions are waiting for locks, with longer queues indicating hotspot conflicts or blocking transactions that need investigation.", "component": "TIKV", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the distribution of Regions and buckets across TiKV instances in the cluster. Essential for identifying Region imbalance issues, uneven data distribution, and potential hotspot problems that could impact cluster performance and stability.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) ", "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"buckets\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Cache OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema cache operations in TiDB, broken down by action and type. Helps diagnose schema loading performance issues, cache hit/miss patterns, and identify excessive schema reloads that could impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infocache_counters{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (action,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 391, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "In-memory scan lock read duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of in-memory lock scanning operations during pessimistic transaction processing in TiKV. Critical for diagnosing performance issues with pessimistic lock checks, especially when p99 or p9999 latencies spike, indicating contention or slow lock table scans that can block transaction commits.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "QPS", "group": "Cluster", "used_for": "Used for monitoring the queries per second (QPS) handled by each TiKV instance, broken down by gRPC message type. Essential for identifying which TiKV nodes are experiencing high request loads and understanding the distribution of different operation types (e.g., Get, Scan, Prewrite) across the cluster.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 393, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiKV, broken down by worker name. Essential for identifying worker thread bottlenecks, uneven task distribution, and diagnosing performance issues related to specific background operations like apply, snapshot, or scheduler workers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 16, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring how long each TiKV instance has been running since its last restart. Essential for tracking instance stability, identifying recent restarts or crashes, and verifying cluster availability during troubleshooting or maintenance operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",component=~\".*tikv\"}\n \n)) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572012, "dashboard": "TiDB-Cluster-Clinic", "title": "Infoschema v2 Cache Operation", "group": "Schema Load", "used_for": "Used for monitoring the performance of TiDB's Infoschema v2 cache operations, including hit rates and operation types. Essential for diagnosing schema metadata access efficiency and identifying cache misses that could impact query planning performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hit\"}[2m]))/(sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hit\"}[2m]))+sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"miss\"}[2m])))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance. Essential for identifying leader distribution imbalances across the cluster, which can indicate scheduling issues or uneven load distribution that may impact performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572014, "dashboard": "TiDB-Cluster-Clinic", "title": "Infoschema v2 Cache Size", "group": "Schema Load", "used_for": "Used for monitoring the memory usage of TiDB's information schema v2 cache against its configured limit. Essential for diagnosing schema metadata memory consumption issues and determining if cache size adjustments are needed to optimize schema loading performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_infoschema_v2_cache_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "tidb_domain_infoschema_v2_cache_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 394, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks queued in TiKV's various worker threads (such as scheduler, apply, and raftstore workers). Essential for identifying thread pool saturation and task processing bottlenecks that can lead to request timeouts and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572002, "dashboard": "TiDB-Cluster-Clinic", "title": "Lease Duration", "group": "Schema Load", "used_for": "Used for monitoring the remaining time until TiDB's schema lease expires. Critical for diagnosing schema synchronization issues and detecting potential schema load problems when the lease duration becomes abnormally short or approaches zero, which could indicate delays in schema information updates across the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} - time()" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572015, "dashboard": "TiDB-Cluster-Clinic", "title": "TableByName API Duration", "group": "Schema Load", "used_for": "Used for monitoring the latency of TableByName API calls in TiDB's schema information cache, tracking p80, p99, and p999 percentiles. Essential for diagnosing slow schema lookups that can impact query planning performance, especially when applications frequently access table metadata or experience schema cache misses.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) /\nsum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)", "histogram_quantile(0.999, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 20, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiKV instances, categorized by failure type and store. Critical for diagnosing cluster health issues, detecting communication problems between TiKV nodes, and identifying stores experiencing reporting failures that could impact cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance, store_id) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 19, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiKV server overload conditions by tracking scheduler busy errors, channel full events, and coprocessor queue saturation rates. Critical for diagnosing when TiKV instances are overwhelmed by excessive workload, helping identify capacity bottlenecks and the need for load balancing or scaling.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Worker Duration 99", "group": "DDL", "used_for": "Used for monitoring the 99th percentile latency of DDL worker operations in TiDB, broken down by operation type and action. Critical for diagnosing slow schema changes and identifying DDL performance bottlenecks that may impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, action, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 62, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Waiting Jobs Count", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs waiting in the queue across TiDB instances. Essential for identifying DDL operation backlogs and potential schema change bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 64, "dashboard": "TiDB-Cluster-Clinic", "title": "Deploy Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL syncer deployment operations in TiDB. Helps diagnose performance issues during schema synchronization setup, particularly when DDL changes are being distributed across the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 395, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiKV's FuturePool thread pools. Helps identify thread pool utilization patterns and diagnose potential bottlenecks when async task processing becomes saturated or unbalanced across different pools.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Backfill Data Rate", "group": "DDL", "used_for": "Used for monitoring the rate at which DDL operations backfill data during index creation in TiDB. Essential for tracking DDL performance and diagnosing slow index building operations, helping identify bottlenecks when adding indexes to large tables.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_add_index_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 396, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks in TiKV's FuturePool thread pools. Helps identify thread pool saturation and task queuing issues that can lead to request processing delays and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Duration 95", "group": "DDL", "used_for": "Used for monitoring the 95th percentile latency of DDL (Data Definition Language) operations in TiDB, including schema changes and index creation. Critical for diagnosing slow DDL performance issues and identifying operations that may be blocking or impacting cluster availability during schema modifications.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 399, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of PD (Placement Driver) requests from TiKV instances. Essential for diagnosing communication issues between TiKV and PD, identifying slow PD operations, and troubleshooting cluster metadata access performance problems.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 403, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD forward status", "group": "PD", "used_for": "Used for monitoring whether TiKV's PD requests are being forwarded to other PD nodes. Helps diagnose PD leader election issues, network connectivity problems between TiKV and PD, or situations where TiKV is communicating with a non-leader PD node that must forward requests.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV instances, broken down by error reason. Essential for diagnosing issues with coprocessor operations (like table scans and aggregations) and identifying specific failure patterns such as lock conflicts, region errors, or timeout issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 401, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiKV and PD, categorized by validation type. Helps diagnose cluster topology issues, region metadata inconsistencies, and potential communication problems between TiKV nodes and the PD scheduler.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 21, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiKV's Raftstore engine. Critical for detecting storage layer errors, consensus failures, or I/O issues that could impact data reliability and cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 18, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiKV instances, categorized by error type. Essential for detecting severe system issues like data corruption, I/O failures, or internal panics that require immediate attention and may impact cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 398, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of requests sent from TiKV to PD (Placement Driver). Helps identify communication patterns between TiKV and PD, and diagnose issues when TiKV cannot properly interact with PD for metadata operations, leader elections, or region scheduling.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 404, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending TSO Requests", "group": "PD", "used_for": "Used for monitoring the number of TSO (Timestamp Oracle) requests currently pending in TiKV's PD client queue. Essential for diagnosing timestamp allocation bottlenecks, PD connectivity issues, or when transactions are experiencing delays due to TSO service problems.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 402, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD reconnection", "group": "PD", "used_for": "Used for monitoring the frequency of TiKV reconnections to PD (Placement Driver) servers. Critical for diagnosing network instability, PD availability issues, or connection problems that could impact cluster coordination and metadata operations.", "component": "TIKV", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring the rate of scheduler errors in TiKV, specifically tracking snapshot and prepare write failures. Critical for diagnosing transaction processing issues, storage layer problems, and identifying when TiKV nodes are experiencing difficulties handling read/write requests.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Log Replication Rejected", "group": "Errors", "used_for": "Used for monitoring the rate of rejected Raft log append operations in TiKV instances. Critical for identifying replication bottlenecks, resource pressure, or configuration issues that prevent followers from accepting log entries from the leader.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 265, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Running Job Cnt", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs currently executing in TiDB. Helps identify DDL operation bottlenecks, concurrent schema changes, and potential blocking issues when multiple DDL statements are queued or running simultaneously.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_running_job_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 400, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages sent from TiKV to PD and tracking pending heartbeats that haven't been processed. Essential for diagnosing communication issues between TiKV and PD, detecting network problems, or identifying when PD is overloaded and unable to process heartbeats in a timely manner.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 25, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region drops on each TiKV instance. Critical for detecting leadership instability, network partitions, or node failures that cause regions to lose their leader status and potentially impact cluster availability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 66, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Self Version Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL owner updating its version information in the cluster. Critical for diagnosing DDL coordination issues and detecting delays in schema change propagation across TiDB instances.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 192, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL backfill progress in percentage", "group": "DDL", "used_for": "Used for tracking the completion percentage of DDL backfill operations in TiDB, including index creation and column modifications. Essential for monitoring long-running schema changes and estimating when DDL operations will complete.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"modify_column\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 24, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV instances, broken down by message type. Critical for identifying communication failures between TiKV and other cluster components (TiDB, PD, or other TiKV nodes), which can indicate network issues, resource exhaustion, or service degradation.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 34, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate region size", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiKV, showing p99, p9999, and average values. Essential for identifying region size imbalances, detecting hotspots, and diagnosing issues related to region split/merge operations that could impact I/O performance and storage efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 264, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL job table duration", "group": "DDL", "used_for": "Used for monitoring the 95th percentile duration of DDL job table operations in TiDB. Critical for diagnosing slow DDL execution and identifying performance bottlenecks in schema changes, helping pinpoint whether delays occur during job table reads/writes or metadata operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", }[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 408, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Slow Trend", "group": "Slow Trend Statistics", "used_for": "Used for tracking the slow trend metric in TiKV's Raft store, which indicates when operations are experiencing performance degradation. Helps identify TiKV instances that are becoming slow over time, enabling early detection of performance issues before they impact cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_slow_trend\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 65, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Handle Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the maximum latency of DDL owner operations when handling schema synchronization across TiDB nodes. Critical for diagnosing DDL execution delays and identifying bottlenecks in distributed schema change coordination, especially when DDL operations appear stuck or slow.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 406, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Inspected duration per server", "group": "Slow Trend Statistics", "used_for": "Used for monitoring the 99th percentile duration of Raft store inspection operations per TiKV instance. Critical for diagnosing slow Raft log processing and identifying TiKV nodes experiencing performance degradation in consensus protocol operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_inspect_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 407, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store Slow Score", "group": "Slow Trend Statistics", "used_for": "Used for monitoring the slow score of each TiKV store instance, which indicates the degree of slowness in Raft store operations. Essential for identifying performance degradation in specific TiKV nodes and diagnosing unbalanced load or hardware issues that cause certain stores to lag behind others.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL META OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per minute in TiDB, tracking the rate of schema changes like CREATE, ALTER, and DROP statements. Essential for diagnosing DDL performance issues, identifying schema change bottlenecks, and understanding the frequency of metadata operations across TiDB instances.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 190, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per minute in TiDB, broken down by operation type (e.g., CREATE, ALTER, DROP). Essential for tracking schema change frequency and identifying periods of heavy DDL activity that may impact cluster performance or cause lock contention.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 26, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of regions without a Raft leader on each TiKV instance. Critical for detecting split-brain scenarios, network partitions, or leadership election failures that can cause service unavailability and data access issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572001, "dashboard": "TiDB-Cluster-Clinic", "title": "Add Index Scan Rate", "group": "DDL", "used_for": "Used for monitoring the row scanning throughput (rows per second) during ADD INDEX operations in TiDB at different percentiles (p50, p95, p999). Essential for evaluating index creation performance and identifying whether slow DDL operations are caused by low scan rates or other bottlenecks.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 411, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "QPS of each store", "group": "Slow Trend Statistics", "used_for": "Used for monitoring the queries per second (QPS) handled by each TiKV store instance, derived from slow trend statistics. Helps identify uneven load distribution across stores and detect instances experiencing abnormal throughput patterns that may indicate performance bottlenecks or hardware issues.", "component": "TIKV", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_slow_trend_result_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average number of bytes written per region across TiKV instances. Helps identify uneven write distribution, hotspot regions, and potential load balancing issues that could impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the distribution of bytes written to regions in TiKV storage. Helps identify write hotspots, unbalanced data distribution, and diagnose performance issues caused by regions receiving disproportionate write traffic.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Damaged files", "group": "Errors", "used_for": "Used for monitoring corrupted files detected in TiKV's RocksDB storage engine, tracking both damaged files found and those automatically deleted. Critical for identifying data integrity issues and potential disk corruption that could lead to data loss or cluster instability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 409, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "QPS Changing Trend", "group": "Slow Trend Statistics", "used_for": "Used for tracking the rate of slow operations detected by TiKV's slow trend detection mechanism across instances. Helps identify performance degradation patterns and instances experiencing increasing latency trends that may indicate underlying storage or resource issues.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_slow_trend_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel full events in TiKV instances, broken down by channel type. Critical for identifying internal communication bottlenecks and backpressure issues that can lead to request queuing, performance degradation, or timeout errors.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 33, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate region size", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes across TiKV instances to understand data distribution patterns. Helps identify regions that may be too large and need splitting, or diagnose uneven data distribution that could impact cluster performance and load balancing.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 410, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "AVG Sampling Latency", "group": "Slow Trend Statistics", "used_for": "Used for monitoring the average sampling latency of slow trend statistics in TiKV's Raft store. Helps identify performance degradation trends and diagnose when TiKV instances are experiencing increasing latency patterns that may indicate underlying storage or replication issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_slow_trend_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total size of each Column Family (CF) in TiKV's RocksDB storage engine across instances. Essential for tracking storage space usage, identifying data growth patterns, and diagnosing potential storage capacity issues or imbalances between different column families (default, write, lock, raft).", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 330, "dashboard": "TiDB-Cluster-Clinic", "title": "SubTask Running Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the execution duration of running subtasks in TiDB's distributed task execution framework. Helps identify long-running subtasks and diagnose performance bottlenecks in distributed operations like DDL, import, or analytics queries.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"running\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 326, "dashboard": "TiDB-Cluster-Clinic", "title": "Completed/Total Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for tracking the progress of distributed task execution by showing completed subtasks versus total subtasks for each task type and ID. Essential for monitoring the health and completion status of distributed operations like DDL, import, or backup/restore tasks, helping identify stalled or failing distributed workloads.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"succeed|failed|canceled|reverted|revert_failed\"}) by (task_id, task_type)", "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (task_id, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 413, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent by each TiKV instance. Essential for diagnosing region replication issues, identifying instances sending excessive snapshots due to slow followers or network problems, and detecting potential performance bottlenecks in cluster data synchronization.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [1m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 38, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region across TiKV instances. Helps identify uneven write distribution, hot regions, and potential load balancing issues that could impact write performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 322, "dashboard": "TiDB-Cluster-Clinic", "title": "Task Status", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution and status of distributed tasks in TiDB's distributed execution framework. Helps identify task execution patterns, detect stuck or failed tasks, and troubleshoot issues with distributed operations like DDL, import, or backup/restore jobs.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (status, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 37, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the distribution of keys written per region in TiKV over time. Helps identify write hotspots, unbalanced region workloads, and diagnose performance issues caused by regions handling excessive write operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 329, "dashboard": "TiDB-Cluster-Clinic", "title": "Pending Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for monitoring the number of pending subtasks in TiDB's distributed execution framework, grouped by executor ID. Helps identify task scheduling bottlenecks and unbalanced workload distribution across executors when subtasks accumulate in pending state.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending\"}) by (exec_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 414, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count and distribution of Raft snapshot operations in TiKV, categorized by snapshot type (sending/receiving/applying). Essential for diagnosing region rebalancing issues, slow follower recovery, and identifying bottlenecks in snapshot transfer during cluster scaling or node recovery scenarios.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the number of Raft leader regions actively handling write operations on each TiKV instance. Helps identify write hotspots, unbalanced leader distribution, and instances experiencing heavy write traffic that may need load rebalancing.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 331, "dashboard": "TiDB-Cluster-Clinic", "title": "Subtask Pending Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the time subtasks spend in pending state before execution in TiDB's distributed task framework. Helps identify task scheduling bottlenecks and resource contention issues when subtasks are queued excessively long before processing begins.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"pending\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 415, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Snapshot generation/apply wait duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile wait time before snapshot generation and application operations begin in TiKV's Raft store. Critical for diagnosing delays in snapshot processing that can indicate scheduling bottlenecks, resource contention, or thread pool saturation affecting region replication and recovery performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 416, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiKV, including sending, applying, and generating snapshots during Raft replication. Critical for diagnosing slow region recovery, replica synchronization delays, and identifying performance bottlenecks when TiKV nodes are catching up with data or handling region transfers.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 417, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile size of snapshots generated during TiKV's Raft replication process. Critical for identifying abnormally large snapshots that could indicate data skew, slow followers, or potential network/storage bottlenecks during region synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Written key size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of key sizes being written to TiKV's Raft store. Helps identify unusual write patterns, oversized keys that may impact performance, and diagnose storage efficiency issues related to key size distribution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 332, "dashboard": "TiDB-Cluster-Clinic", "title": "Uncompleted Subtask Distribution on TiDB Nodes", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of incomplete subtasks across TiDB nodes in the distributed execution framework. Helps identify workload imbalance, stuck tasks, or nodes with excessive pending/failed subtasks that may indicate resource bottlenecks or execution issues.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending|running|failed|canceled|paused\"}) by (exec_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 418, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pair counts in TiKV snapshots. Helps identify outlier transactions that access an unusually large number of keys, which can indicate inefficient queries or potential performance bottlenecks in snapshot read operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 41, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Written value size for every put of apply worker", "group": "Server", "used_for": "Used for monitoring the distribution of data value sizes written during Raft apply operations in TiKV. Helps identify write patterns and diagnose issues related to large value writes that may impact apply worker performance and memory usage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 328, "dashboard": "TiDB-Cluster-Clinic", "title": "Slots usage", "group": "Dist Execute Framework", "used_for": "Used for monitoring the resource slot utilization in TiDB's distributed execution framework compared to maximum available processors. Helps identify resource contention and capacity planning issues when distributed tasks consume excessive slots, potentially impacting concurrent task execution performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_used_slots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 419, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot Actions", "group": "Snapshot", "used_for": "Used for monitoring Raft snapshot operations and region cleanup activities in TiKV. Essential for diagnosing snapshot transfer issues, region migration problems, and identifying bottlenecks in Raft replication when nodes are catching up or recovering from failures.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Inaccuracy Rate", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the inaccuracy rate of table statistics in TiDB at different percentiles (p50, p90, p99). Critical for identifying when outdated or inaccurate statistics are causing suboptimal query plans, helping determine when to run ANALYZE TABLE to refresh statistics.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 46, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the duration of automatic statistics collection (auto-analyze) operations in TiDB at the 80th and 95th percentiles. Essential for diagnosing performance issues related to background statistics updates, which can impact query optimization and overall cluster performance when taking too long.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 233, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Healthy Distribution", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the health distribution of table statistics in TiDB, categorized by type (table/index). Essential for identifying stale or missing statistics that can lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_statistics_stats_healthy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 42, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Hibernate Peers", "group": "Server", "used_for": "Used for monitoring the number and state of hibernated Raft peers in TiKV instances. Helps identify regions that have entered hibernation mode to reduce CPU overhead, and diagnose issues related to peer state transitions or unexpected hibernation behavior.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 93, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Stats OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of statistics update operations in TiDB, broken down by update type. Essential for diagnosing statistics collection performance issues and understanding when auto-analyze jobs are running, which can impact query optimization and cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 71, "dashboard": "TiDB-Cluster-Clinic", "title": "Pseudo Estimation OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of queries using pseudo statistics (estimated stats when real statistics are unavailable) in TiDB. High values indicate missing or outdated table statistics, which can lead to suboptimal query plans and performance degradation requiring ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 39, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Clear overlap region duration", "group": "Server", "used_for": "Used for monitoring the latency of clearing overlapping regions during TiKV's region merge operations, tracking both p99 and p9999 percentiles. Critical for diagnosing performance issues during region consolidation, especially when regions are being merged after splits or load rebalancing, as high durations indicate potential bottlenecks in the raftstore's region management process.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO time per second", "group": "Server", "used_for": "Used for monitoring the time spent on disk I/O operations per second in TiKV's RocksDB storage engine, broken down by storage and coprocessor requests. Critical for diagnosing disk performance bottlenecks and identifying whether slow queries are caused by excessive disk read latency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Queries Per Minute", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of automatic statistics analysis operations in TiDB per minute, broken down by analysis type. Essential for tracking the frequency of auto-analyze jobs and diagnosing issues with stale statistics or excessive background analysis activity that may impact cluster performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 46, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile wait time before tasks are scheduled in TiKV's thread pools. Critical for diagnosing thread pool saturation and identifying when tasks are queuing excessively before execution, which can indicate resource contention or insufficient thread pool capacity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory trace", "group": "Server", "used_for": "Used for tracking detailed memory usage breakdown in TiKV, specifically monitoring memory consumption of the Raftstore component and Raft Engine. Essential for diagnosing memory-related issues, identifying memory leaks, and understanding which internal components are consuming the most memory resources in TiKV instances.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 45, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolve address duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of address resolution operations in TiKV servers. Critical for diagnosing network-related issues, DNS resolution delays, or service discovery problems that could impact inter-node communication and overall cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the average time tasks spend waiting in TiKV's thread pool queues before execution. Critical for diagnosing thread pool saturation and identifying when TiKV is overloaded, causing requests to queue excessively before processing begins.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 421, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot pending applies", "group": "Snapshot", "used_for": "**Used For:**\n\nUsed for monitoring the number of Raft snapshots waiting to be applied in TiKV instances. Helps diagnose Region replication delays and identify bottlenecks when followers are slow to catch up with leaders, which can impact cluster recovery time and data consistency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 424, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advance ts Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU usage of the advance timestamp worker threads in TiKV's Resolved TS mechanism. Essential for diagnosing performance issues related to timestamp advancement and identifying instances where the advance-ts worker is consuming excessive CPU resources, which could impact transaction processing and CDC (Change Data Capture) functionality.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Entry Cache Evicts", "group": "Server", "used_for": "Used for monitoring the rate at which Raft log entries are evicted from TiKV's in-memory cache. Helps diagnose memory pressure issues and identify instances where the Raft entry cache is being aggressively cleared, which may impact replication performance and indicate insufficient memory allocation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 426, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts", "group": "Resolved TS", "used_for": "Used for monitoring the maximum gap between current time and resolved timestamp across TiKV instances. Critical for diagnosing CDC (Change Data Capture) replication lag and identifying instances where transaction resolution is delayed, which can impact downstream data consistency and real-time data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 423, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolved TS Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU usage of TiKV's Resolved TS worker threads across instances. Essential for diagnosing performance bottlenecks in the Resolved TS mechanism, which is critical for maintaining transaction consistency and supporting features like Stale Read and CDC (Change Data Capture).", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 49, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO bytes per second", "group": "Server", "used_for": "Used for monitoring the rate of disk I/O operations in bytes per second for TiKV's RocksDB storage engine, broken down by storage and coprocessor requests. Essential for diagnosing disk throughput bottlenecks, identifying I/O-intensive workloads, and correlating storage performance issues with query patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 229, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load QPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the throughput of synchronous statistics loading operations in TiDB, including total load requests, timeout occurrences, and deduplicated requests. Essential for diagnosing query plan quality issues caused by missing or stale statistics, and identifying when statistics loading becomes a performance bottleneck.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_dedup_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 420, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot transport speed", "group": "Snapshot", "used_for": "Used for monitoring the speed of snapshot data transfer and generation across TiKV instances. Essential for diagnosing slow region replication, identifying network bottlenecks during snapshot transport, and detecting instances that are struggling to generate or receive snapshots during cluster scaling or recovery operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 234, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache Cost", "group": "Statistics & Plan Management", "used_for": "Used for monitoring TiDB's statistics cache memory usage, tracking both current memory consumption and configured capacity limits. Essential for diagnosing memory pressure issues related to table statistics caching and optimizing cache configuration to balance query planning performance with memory overhead.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"track\"}", "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"capacity\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 235, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per second on TiDB's statistics cache, broken down by operation type (hits, misses, evictions). Essential for diagnosing query optimizer performance issues and determining if statistics cache sizing is adequate for workload demands.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_stats_cache_op{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 428, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of follower safe-ts", "group": "Resolved TS", "used_for": "Used for monitoring the maximum time gap between follower replicas' safe timestamps in TiKV's Resolved TS mechanism. Critical for diagnosing data replication lag and identifying followers that are falling behind in maintaining consistent read timestamps across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 429, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Safe TS Follower Region", "group": "Resolved TS", "used_for": "Used for monitoring which TiKV region has the minimum safe timestamp among follower replicas on each instance. Critical for diagnosing Resolved TS lag issues and identifying regions that are blocking transaction consistency guarantees in distributed transactions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 51, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages processed by TiKV instances, broken down by message type and priority. Essential for identifying communication patterns between TiDB/TiKV components, detecting abnormal RPC traffic, and diagnosing issues related to specific gRPC operations like Coprocessor requests, prewrite, or commit operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 425, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scan lock Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU usage of TiKV's incremental scan lock workers, which are responsible for scanning locks during Resolved TS advancement. Essential for diagnosing performance bottlenecks in CDC (Change Data Capture) or follower read scenarios where high CPU usage may indicate lock scanning overhead or slow Resolved TS progress.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 427, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Resolved TS Region", "group": "Resolved TS", "used_for": "Used for identifying which TiKV region has the minimum (most lagging) resolved timestamp across instances. Critical for diagnosing Resolved TS lag issues that can impact CDC replication, backup progress, and stale read consistency, helping pinpoint problematic regions blocking timestamp advancement.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 236, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Replayer Task OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring Plan Replayer task execution rates in TiDB, tracking both dump and capture operations per minute along with registered task counts. Essential for diagnosing issues with SQL plan reproduction and troubleshooting query plan problems, helping identify failures in plan capture or export operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)", "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"capture\"}[2m])) by (result)", "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 230, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load Latency P9999", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99.99th percentile latency of synchronous statistics loading operations in TiDB, including both overall sync load time and stats reading time. Critical for diagnosing extreme tail latency issues in query optimization when TiDB needs to load table statistics on-demand, which can impact query planning performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 40, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the rate of read tasks processed per second by each TiFlash instance. Helps identify read workload distribution across TiFlash nodes and diagnose performance bottlenecks when read throughput is unbalanced or unexpectedly low.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 19, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Internal DDL OPM", "group": "DDL", "used_for": "Used for monitoring the operations per minute (OPM) of internal DDL schema changes in TiFlash, broken down by DDL type and instance. Essential for tracking TiFlash's schema synchronization activity and diagnosing issues with DDL replication from TiDB to TiFlash replicas.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring the uptime duration of TiFlash-Proxy instances since their last restart. Helps identify recent restarts, stability issues, or instances that may have crashed and recovered, which is critical for troubleshooting cluster reliability problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - tiflash_proxy_process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 226, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Count", "group": "Pipeline Model", "used_for": "Used for monitoring the number of pipeline tasks in TiFlash's execution engine, showing both per-instance and cluster-wide task counts. Essential for diagnosing TiFlash query execution bottlenecks, identifying task accumulation issues, and understanding workload distribution across instances.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (instance, type)", "sum(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 103, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads", "group": "Coprocessor", "used_for": "Used for monitoring the maximum thread count limits configured in TiFlash instances across different thread pools. Helps identify thread pool capacity constraints and potential resource bottlenecks when actual thread usage approaches these maximum thresholds.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 123, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "0.99 Duration of raft store events", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile latency of Raft store events in TiFlash-Proxy, tracking how long different Raft operations take to complete. Critical for diagnosing TiFlash replication delays and identifying performance bottlenecks in Raft log processing, particularly when investigating slow data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 96, "dashboard": "DM-Professional-Clinic", "title": "Chunk process duration", "group": "dump and load ", "used_for": "Used for monitoring the average time taken to process each data chunk during DM's dump and load operations. Essential for identifying performance bottlenecks in data migration tasks and diagnosing slow chunk processing that may indicate resource constraints or inefficient data handling.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "1/rate(lightning_chunks{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",state=\"finished\"}[2m])" ], "dashboard_file": "dm.json" }, { "panel_id": 18, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply OPM", "group": "DDL", "used_for": "Used for monitoring the rate of schema changes (DDL operations) being applied to TiFlash instances, measured in operations per minute. Helps identify DDL activity patterns and diagnose issues when schema synchronization between TiDB and TiFlash is slow or failing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Messages", "group": "Raft message", "used_for": "Used for monitoring the rate of different types of Raft messages sent by TiFlash-Proxy nodes in the cluster. Helps diagnose Raft communication patterns and identify potential issues with consensus protocol message flow, such as excessive heartbeats, vote requests, or append entries that could indicate network problems or cluster instability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1616, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft message flushes per TiFlash-Proxy server instance. Helps identify network communication patterns and potential bottlenecks in Raft consensus message processing between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 166, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data size in send and receive queue", "group": "Coprocessor", "used_for": "Used for monitoring the amount of data queued in TiFlash's exchange operators during coprocessor request processing. Helps identify data transfer bottlenecks and memory pressure when TiFlash nodes exchange intermediate results during distributed query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_exchange_queueing_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 40, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations in TiFlash-Proxy per server instance. Critical for diagnosing write performance bottlenecks and identifying slow disk I/O issues in TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "DM-Professional-Clinic", "title": "SQL process rate", "group": "dump and load ", "used_for": "Used for monitoring the SQL data processing throughput rate during DM's dump and load operations. Helps identify data ingestion bottlenecks and track the speed at which SQL statements are being delivered and processed across different tasks and sources.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 199, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Time of the Longest Live MPP Task", "group": "Coprocessor", "used_for": "Used for monitoring the execution time of the longest-running MPP (Massively Parallel Processing) task currently active in TiFlash. Helps identify stuck or abnormally long-running distributed query tasks that may indicate performance bottlenecks or resource contention in the MPP execution engine.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_mpp_task_monitor{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 41, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of write operations in TiFlash storage engine, tracking both storage command counts and DM write block events per second. Essential for identifying write workload patterns, detecting write traffic spikes, and diagnosing TiFlash write performance bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1615, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages sent from each TiFlash-Proxy instance to other nodes in the cluster. Helps identify network communication bottlenecks, unbalanced message distribution, or instances experiencing Raft replication issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4, "dashboard": "DM-Professional-Clinic", "title": "binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being processed per second during DM (Data Migration) replication. Essential for tracking replication throughput and identifying bottlenecks or slowdowns in the binlog transformation pipeline across different tasks and stages.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "interval", "job_id", "source", "stage", "task" ], "promql": [ "sum by (instance,task,stage) (rate(dm_syncer_binlog_transform_cost_count{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\", source_id=~\"$source\",stage=~\"$stage\"}[$interval]))" ], "dashboard_file": "dm.json" }, { "panel_id": 20, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL schema changes being applied to TiFlash replicas at various percentiles (p80-p999). Critical for diagnosing slow DDL propagation issues and identifying when schema synchronization between TiDB and TiFlash becomes a bottleneck, which can impact query performance on analytical workloads.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "type" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", keyspace!=\"\"}[2m])) by (le, type, keyspace))", "sum(tiflash_sync_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"$type\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft propose", "used_for": "Used for monitoring the rate of different types of Raft proposals in TiFlash-Proxy, including read operations (local_read, read_index) and write operations (normal, batch). Essential for diagnosing I/O bottlenecks and understanding the distribution of read/write workload patterns in TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 222, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Duration", "group": "Pipeline Model", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of pipeline tasks in TiFlash's pipeline execution model. Essential for identifying performance bottlenecks in TiFlash query processing and diagnosing slow task execution issues that impact overall query latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m]))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 117, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Hard Limit Exceeded Count", "group": "Task Scheduler", "used_for": "Used for monitoring how often TiFlash task scheduler hits hard resource limits by resource group and instance. Critical for identifying resource exhaustion issues and determining when task scheduling is being throttled due to insufficient capacity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hard_limit_exceeded_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 109, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Estimated Thread Usage and Limit", "group": "Task Scheduler", "used_for": "Used for monitoring TiFlash's task scheduler thread consumption against configured soft and hard limits per resource group. Critical for identifying thread exhaustion issues, resource group contention, and preventing task scheduler overload that could impact query execution performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_soft_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_hard_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"global_estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"group_entry_count\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3690, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of committing Raft logs to disk in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing write path performance issues and identifying when disk I/O bottlenecks are affecting TiFlash's ability to replicate data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 37, "dashboard": "DM-Professional-Clinic", "title": "Remaining time to sync", "group": "binlog replication", "used_for": "Used for monitoring the estimated time remaining for DM (Data Migration) to catch up with the upstream binlog replication lag. Critical for tracking replication progress and identifying when sync delays may impact data freshness or recovery objectives.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_remaining_time{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 111, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Queries Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting queries in TiFlash's task scheduler across different resource groups. Essential for identifying query queuing issues, resource contention, and determining if TiFlash is overloaded or experiencing scheduling bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_queries_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_queries_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 216, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of Wait Reactor", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's Wait Reactor threads, which handle asynchronous I/O operations and event processing in the pipeline execution model. Essential for diagnosing whether wait reactors are becoming CPU bottlenecks during query execution or experiencing thread contention issues.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 246, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "TiFlash Resource Group", "group": "TiFlash Resource Control", "used_for": "Used for monitoring TiFlash resource group consumption and throttling behavior, including remaining tokens, average consumption speed, and total resource usage per group. Essential for diagnosing resource control effectiveness, identifying resource-constrained groups, and ensuring fair resource allocation across different workloads in TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"remaining_tokens\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"avg_speed\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"total_consumption\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_fill_rate\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_capacity\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fetch_tokens_from_gac_count\", instance=~\"$instance\"}) by (instance,resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 60, "dashboard": "DM-Professional-Clinic", "title": "transaction/ddl query execution latency", "group": "binlog replication", "used_for": "Used for monitoring the P90, P95, and P99 latency of transaction and DDL query execution during binlog replication in DM (Data Migration). Critical for identifying replication lag issues and diagnosing slow transaction processing that could impact data synchronization performance between upstream and downstream databases.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.90, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 1309, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft message", "used_for": "Used for monitoring the rate of dropped Raft messages in TiFlash-Proxy, categorized by message type. Critical for diagnosing Raft communication issues, network problems, or overload conditions that prevent proper consensus protocol operation between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 89, "dashboard": "DM-Professional-Clinic", "title": "External resources", "group": "dump and load ", "used_for": "Used for monitoring the number of active external resources (KV encoders and importer engines) during DM's dump and load operations. Helps identify resource leaks or bottlenecks by tracking the difference between opened and closed resources in the Lightning component.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}", "lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 107, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Min TSO", "group": "Task Scheduler", "used_for": "Used for monitoring the minimum Timestamp Oracle (TSO) value being processed by TiFlash's task scheduler across different resource groups. Helps identify data freshness issues and diagnose replication lag between TiKV and TiFlash, as lower TSO values indicate older data being processed.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"min_tso\"}) by (instance, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1710, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring disk I/O utilization percentage across TiFlash-Proxy nodes. Essential for identifying disk bottlenecks and determining if storage I/O capacity is saturated, which can impact TiFlash query performance and data synchronization.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 115, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Waiting Duration", "group": "Task Scheduler", "used_for": "Used for monitoring the waiting time of tasks in TiFlash's task scheduler at p80, p90, and p100 percentiles. Essential for diagnosing query queuing issues and resource contention when TiFlash queries experience delays before execution starts.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(1.00, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 32, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs in TiFlash-Proxy per server instance. Critical for diagnosing replication lag and identifying TiFlash nodes experiencing slow Raft log application, which can impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 113, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Tasks Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting tasks in TiFlash's task scheduler across different resource groups. Helps identify task queue bottlenecks, resource contention, and scheduling inefficiencies when tasks accumulate in waiting state or when active task counts approach capacity limits.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_tasks_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_tasks_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 38, "dashboard": "DM-Professional-Clinic", "title": "Replication lag", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile replication lag in DM (Data Migration) binlog synchronization tasks. Critical for identifying data synchronization delays between upstream and downstream databases, helping diagnose replication bottlenecks and ensuring data consistency within acceptable time windows.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_lag_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,source_id,task))", "dm_syncer_replication_lag_gauge{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 31, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of applying Raft logs in TiFlash-Proxy's storage engine, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication lag and identifying bottlenecks in the Raft log application process that could impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 5, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft process", "used_for": "Used for monitoring the rate of Raft ready operations handled by TiFlash-Proxy, broken down by operation type. Essential for diagnosing Raft consensus processing performance and identifying bottlenecks in TiFlash's replication pipeline when data synchronization from TiKV appears slow or stalled.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 258, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (bytes)", "group": "Storage", "used_for": "Used for monitoring the write throughput of TiFlash storage subtasks, broken down by operation type. Helps identify write performance bottlenecks and understand which storage operations are consuming the most bandwidth in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 186, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Flow", "group": "S3", "used_for": "Used for monitoring the rate of data flow through TiFlash's remote cache (S3-backed cache), broken down by operation type. Helps diagnose remote cache efficiency, identify cache hit/miss patterns, and troubleshoot performance issues related to S3 object storage access in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 64, "dashboard": "DM-Professional-Clinic", "title": "DML conflict detect duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of DML conflict detection during binlog replication in DM (Data Migration) tasks at p90, p95, and p99 percentiles. Critical for diagnosing replication slowdowns caused by conflict detection overhead, especially when multiple concurrent writes target the same data and require conflict resolution.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 110, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS (Sum by task)", "group": "binlog replication", "used_for": "Used for monitoring the binlog replication throughput in DM (Data Migration) tasks, showing both actual finished jobs per second and ideal QPS aggregated by task. Essential for identifying replication performance bottlenecks, comparing actual versus expected throughput, and diagnosing slow data synchronization issues across different migration tasks.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "task" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (task)", "sum(dm_syncer_ideal_qps{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 38, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Amplification", "group": "Storage", "used_for": "Used for monitoring the write amplification ratio in TiFlash storage, which measures how much data is actually written to disk compared to the logical data received. Critical for diagnosing storage efficiency issues and identifying excessive disk I/O that can impact TiFlash performance and disk lifespan.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[10m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[30m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 230, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Max Execute Time Per Round", "group": "Pipeline Model", "used_for": "Used for monitoring the maximum execution time of TiFlash pipeline tasks per scheduling round at p95 and p99 percentiles. Critical for identifying pipeline task scheduling inefficiencies, detecting tasks that monopolize execution time, and diagnosing TiFlash query performance degradation caused by unbalanced task execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3688, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations in TiFlash-Proxy per server instance. Critical for diagnosing write path performance issues and identifying slow disk I/O or consensus bottlenecks that could impact TiFlash data replication and consistency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 228, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Status Change OPS", "group": "Pipeline Model", "used_for": "Used for monitoring the rate of pipeline task status transitions in TiFlash's pipeline execution model. Helps diagnose task scheduling efficiency and identify potential bottlenecks when tasks frequently change states, indicating possible resource contention or execution issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_pipeline_task_change_to_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\r\n" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 63, "dashboard": "DM-Professional-Clinic", "title": "add job duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of adding binlog replication jobs to DM syncer queues at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when job queueing delays impact replication performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "queueNo", "source", "task", "type" ], "promql": [ "histogram_quantile(0.90, sum (rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.95, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.99, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))" ], "dashboard_file": "dm.json" }, { "panel_id": 220, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of IO Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's IO task thread pool, which handles asynchronous I/O operations. Essential for identifying CPU bottlenecks in I/O processing and determining if the IO thread pool is saturated or underutilized during heavy read/write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 224, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Thread Pool Size", "group": "Pipeline Model", "used_for": "Used for monitoring the size of task thread pools in TiFlash's pipeline execution model. Helps diagnose resource allocation issues and thread pool capacity problems when investigating query execution performance or I/O bottlenecks in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_task_thread_pool_size\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile of Raft proposals processed per ready batch in TiFlash-Proxy. Helps identify Raft log application efficiency and potential bottlenecks when TiFlash nodes experience slow data replication or high I/O latency during Raft consensus operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "DM-Professional-Clinic", "title": "Idle workers", "group": "dump and load ", "used_for": "Used for monitoring the number of idle worker threads in DM's lightning component during data dump and load operations. Helps identify resource utilization issues, such as insufficient parallelism or bottlenecks preventing workers from processing data efficiently.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_idle_workers{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages received by each TiFlash-Proxy instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster coordination issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 39, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "binlog replication", "used_for": "Used for monitoring abnormal process terminations in DM's binlog replication component. Helps quickly identify and diagnose replication failures by tracking error-triggered exits over 30-minute intervals, essential for maintaining data migration stability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_syncer_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 1, "dashboard": "DM-Professional-Clinic", "title": "statement execution latency - 90", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile execution latency of binlog replication statements in DM (Data Migration), broken down by transaction phases (begin, statement execution, and commit). Critical for identifying performance bottlenecks in the replication pipeline and diagnosing slow transaction processing that affects data synchronization lag.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"begin\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"stmt\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"commit\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"rollback\", instance=~\"$instance\"}[2m])) by (le,task,instance))" ], "dashboard_file": "dm.json" }, { "panel_id": 35, "dashboard": "DM-Professional-Clinic", "title": "skipped binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being skipped during DM (Data Migration) replication, broken down by rows and query types. Essential for identifying data filtering effectiveness and diagnosing potential data synchronization gaps when binlog filter rules are applied.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"rows\"}[$interval])", "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"query\"}[$interval])" ], "dashboard_file": "dm.json" }, { "panel_id": 118, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Process ready duration per server", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile duration of Raft ready state processing in TiFlash-Proxy per server. Critical for diagnosing Raft replication latency issues and identifying servers experiencing slow consensus processing that could impact TiFlash data synchronization performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 188, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Usage", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache utilization when using S3-based disaggregated storage, tracking both DTFile cache and page cache capacity versus actual usage. Essential for diagnosing cache efficiency issues and determining if cache sizing is appropriate for workload performance in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_DTFileCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DTFileCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2, "dashboard": "DM-Professional-Clinic", "title": "DM SQL jobs ", "group": "binlog replication", "used_for": "Used for monitoring the rate of SQL job processing in DM's binlog replication pipeline, showing both completed jobs (finished) and incoming jobs (added) per second. Essential for identifying replication bottlenecks, job queue buildup, and understanding the throughput of different SQL operation types (INSERT/UPDATE/DELETE) across replication tasks.", "component": "", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", "sum(-rate(dm_syncer_added_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 1709, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring the resident memory usage of TiFlash-Proxy processes across cluster instances. Essential for tracking memory consumption patterns, detecting memory leaks, and ensuring TiFlash-Proxy operates within acceptable memory limits to prevent OOM issues.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 130, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of TiFlash's internal storage maintenance operations, including delta merge, segment merge, and segment split tasks. Essential for diagnosing TiFlash storage performance issues and understanding background compaction activity that may impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "File Open OPS", "group": "Storage", "used_for": "Used for monitoring the rate of file open operations and failures in TiFlash storage layer. Helps diagnose file descriptor exhaustion, storage I/O issues, or excessive file access patterns that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 43, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM syncer during data migration. Critical for identifying replication lag and potential data synchronization delays, helping diagnose whether the syncer is falling behind in processing binlog events.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"master\"} - ON(task,source_id) dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 119, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft read proposals (including local reads and read index operations) per TiFlash instance. Helps identify read request distribution across TiFlash nodes and diagnose imbalanced read workloads or potential read performance bottlenecks in the Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2741, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiFlash-Proxy instances, categorized by error type. Essential for detecting severe operational issues that could impact TiFlash's ability to replicate and serve data, requiring immediate investigation and intervention.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1975, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft log speed", "group": "Raft propose", "used_for": "Used for monitoring the average rate of Raft log data being proposed in TiFlash-Proxy instances, measured in bytes per second. Helps identify write throughput patterns and potential bottlenecks in Raft log replication to TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1584, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiFlash-Proxy server overload conditions, including scheduler busy errors, channel saturation, and coprocessor request rejections. Critical for identifying when TiFlash-Proxy instances are overwhelmed by workload, helping diagnose performance degradation and request failures due to resource exhaustion.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 50, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Opened File Count", "group": "Storage", "used_for": "Used for monitoring the number of open file descriptors in TiFlash instances, including total open files, files opened for writing, and files opened for reading. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiFlash crashes or performance degradation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft Entry Cache", "group": "Cluster", "used_for": "Used for monitoring memory usage of the Raft entry cache in TiFlash-Proxy's raftstore. Helps identify memory pressure issues and potential memory leaks in the Raft log replication layer when TiFlash nodes experience high memory consumption or OOM problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiFlash-Proxy nodes, categorized by failure type and instance. Critical for identifying communication issues between TiFlash-Proxy and PD/TiKV, which can indicate network problems, node unavailability, or cluster coordination failures affecting query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1714, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the total number of Raft regions managed by each TiFlash-Proxy instance. Essential for tracking region distribution across TiFlash nodes and diagnosing data skew or unbalanced region allocation issues that could impact query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 45, "dashboard": "DM-Professional-Clinic", "title": "unsynced tables", "group": "binlog replication", "used_for": "Used for monitoring the number of tables that have not yet been synchronized in DM (Data Migration) binlog replication tasks. Critical for identifying replication lag at the table level and diagnosing incomplete data migration scenarios.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_unsynced_table_number{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 42, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum latency of internal storage subtasks in TiFlash, such as delta merge operations and background compaction tasks. Critical for diagnosing TiFlash storage engine performance issues and identifying bottlenecks in background maintenance operations that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 58, "dashboard": "DM-Professional-Clinic", "title": "binlog event cost duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of binlog event transformation at different stages (p90, p95, p99 percentiles) in DM (Data Migration) workers. Essential for diagnosing binlog replication bottlenecks and identifying which transformation stage is causing delays when data migration performance degrades.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))" ], "dashboard_file": "dm.json" }, { "panel_id": 47, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Read OPS", "group": "Storage", "used_for": "Used for monitoring the rate of disk read operations in TiFlash storage layer, including PSM (PageStorage Manager) read I/O calls, page reads, and file descriptor reads. Essential for diagnosing TiFlash read performance bottlenecks and identifying excessive disk I/O that may impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2535, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft log entries are applied in TiFlash-Proxy's raftstore, tracking p95, p99, and average latencies. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1719, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring error rates in TiFlash-Proxy's scheduler component, specifically tracking snapshot and prepare write failures. Critical for diagnosing data replication issues between TiKV and TiFlash, helping identify when the scheduler fails to process write operations or snapshot requests.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 17, "dashboard": "DM-Professional-Clinic", "title": "Total bytes for load unit", "group": "dump and load ", "used_for": "Used for monitoring the total data volume processed during DM's load phase, tracking both bytes read from source files and bytes delivered to TiDB. Essential for understanding data migration progress, estimating completion time, and diagnosing throughput bottlenecks in the load unit.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_row_read_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", "lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 259, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (rows)", "group": "Storage", "used_for": "Used for monitoring the row-level write throughput of TiFlash storage subtasks, broken down by operation type. Helps identify write performance bottlenecks and understand the distribution of different write operations (such as delta merge, compaction) in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 44, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between relay and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the relay log and syncer components in DM (Data Migration). Helps identify replication lag by showing how many binlog files the syncer is behind the relay, which is critical for diagnosing data synchronization delays and potential bottlenecks in the replication pipeline.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id", "task" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"} - ON(instance) group_right dm_syncer_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1721, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by instance and message type. Essential for diagnosing communication failures between TiFlash and TiKV, helping identify network issues or protocol-level errors that could impact data replication and query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 107, "dashboard": "DM-Professional-Clinic", "title": "Flushed Checkpoint Time Interval", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile time interval between checkpoint flushes during binlog replication in DM (Data Migration). Critical for diagnosing checkpoint persistence delays that could impact data consistency and recovery point objectives during replication failures or restarts.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_flush_checkpoints_time_interval_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task,worker))" ], "dashboard_file": "dm.json" }, { "panel_id": 218, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of CPU Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's CPU task thread pool, which handles compute-intensive operations in the pipeline execution model. Essential for identifying CPU bottlenecks, thread pool saturation, and optimizing TiFlash query performance under heavy analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 46, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Write OPS", "group": "Storage", "used_for": "Used for monitoring disk write operations per second in TiFlash storage layer, including PSM (PageStorage Manager) write calls, page writes, and file descriptor writes. Essential for diagnosing TiFlash write throughput bottlenecks and identifying abnormal disk I/O patterns that may impact columnar storage performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 106, "dashboard": "DM-Professional-Clinic", "title": "binlog event row number", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile of row counts per binlog event during DM (Data Migration) replication. Helps identify binlog events with unusually large row batches that may cause replication lag or memory pressure in the sync unit.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_row_bucket{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source))" ], "dashboard_file": "dm.json" }, { "panel_id": 4375, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile latency of performance context operations during Raft proposal processing in TiFlash-Proxy, including both store and apply stages. Critical for diagnosing slow Raft log replication and identifying performance bottlenecks in TiFlash's consensus layer that may impact write throughput and data synchronization delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 41, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft proposals are processed in TiFlash-Proxy's raftstore, showing p95, p99, and average latencies. Critical for diagnosing Raft replication bottlenecks and identifying when proposals are queuing excessively before being handled, which can indicate raftstore thread saturation or resource contention.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 39, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of appending Raft logs to disk in TiFlash-Proxy's Raft store, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication performance issues and identifying disk I/O bottlenecks that could slow down data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 46, "dashboard": "DM-Professional-Clinic", "title": "shard lock resolving", "group": "binlog replication", "used_for": "Used for monitoring the number of shard DDL locks currently being resolved in DM (Data Migration) during binlog replication. Essential for diagnosing coordination issues when merging sharded tables, especially when DDL operations are stuck or taking longer than expected across multiple upstream MySQL instances.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_shard_lock_resolving{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1723, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions in TiFlash-Proxy that are missing their leaders. Critical for diagnosing replication issues and cluster instability, as missing leaders indicate regions that cannot serve read/write requests until leader election completes.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 60, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write flow", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash's write throughput across different write paths, including file descriptor writes, PageStorage (PSM) writes, and background writes. Essential for diagnosing TiFlash storage I/O bottlenecks, identifying write-heavy workloads, and understanding data ingestion performance from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1722, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region drops in TiFlash-Proxy instances. Essential for detecting leadership instability, network partitions, or node failures that cause unexpected leader transfers, which can impact query performance and data availability.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed in TiFlash-Proxy's raftstore. Critical for diagnosing replication delays and identifying bottlenecks in the Raft consensus pipeline when TiFlash experiences slow data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 65, "dashboard": "DM-Professional-Clinic", "title": "skip event duration", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the latency of skipping binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing performance issues when DM filters or skips specific binlog events based on configured rules, helping identify if event filtering is causing replication delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 4194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiFlash's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate workload imbalances or priority queue bottlenecks affecting read performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 33, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "Used for monitoring the total storage size of different column families (CFs) in TiFlash-Proxy's RocksDB engine. Helps track storage usage patterns and identify potential disk space issues or data growth anomalies in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 67, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Current Data Management Tasks", "group": "Storage", "used_for": "Used for monitoring the number of active data management tasks in TiFlash's DeltaTree storage engine, including delta-merge operations, segment splits, and segment merges. Essential for understanding TiFlash's background compaction workload and diagnosing storage performance issues or resource contention during data reorganization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 71, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiFlash's Raft proxy layer. Critical for identifying extreme tail latency in split operations that could indicate storage bottlenecks or resource contention affecting TiFlash's data distribution and rebalancing performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 131, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum duration of large internal storage tasks in TiFlash, such as delta merge, delta merge background, and place index operations. Critical for diagnosing TiFlash storage layer performance issues and identifying when background maintenance tasks are taking unusually long to complete.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 105, "dashboard": "DM-Professional-Clinic", "title": "replication transaction batch", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile batch size of replicated transactions in DM's binlog replication process. Helps identify transaction batching efficiency and diagnose replication throughput issues when batch sizes are unexpectedly small or large.", "component": "", "phenomenon": "io", "object": "transaction", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_transaction_batch_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task,queueNo,type))" ], "dashboard_file": "dm.json" }, { "panel_id": 120, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft write proposals per TiFlash node. Helps identify uneven write distribution across TiFlash instances and diagnose replication bottlenecks or nodes experiencing abnormal Raft proposal activity.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 62, "dashboard": "DM-Professional-Clinic", "title": "DML queue remain length", "group": "binlog replication", "used_for": "Used for monitoring the number of DML (Data Manipulation Language) events queued and waiting to be replicated in DM's binlog synchronization pipeline. Critical for identifying replication bottlenecks and backlog issues when the queue length grows, indicating that DML processing cannot keep up with the incoming binlog events.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "task" ], "promql": [ "sum(dm_syncer_queue_size{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 237, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexError", "group": "Storage", "used_for": "Used for monitoring the rate of delta index errors in TiFlash's DeltaTree storage engine. Critical for identifying data integrity issues or corruption in TiFlash's columnar storage that could affect query accuracy and system stability.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied in TiFlash-Proxy's Raftstore, broken down by server instance. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 52, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FSync Status", "group": "Storage", "used_for": "Used for monitoring file synchronization (fsync) operations in TiFlash storage, tracking both the rate of fsync calls and their latency distribution. Critical for diagnosing disk I/O performance issues and ensuring data durability, as excessive fsync latency can indicate storage bottlenecks affecting TiFlash write performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 36, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of reading binlog events from upstream MySQL sources in DM (Data Migration) replication tasks. Critical for diagnosing binlog reading performance issues and identifying when network latency or source database load impacts replication lag at the p90, p95, and p99 percentiles.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 61, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the size distribution of binlog events being replicated by DM (Data Migration) workers at p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that may cause replication delays, memory pressure, or network bottlenecks during data synchronization from MySQL to TiDB.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,worker,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 4196, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of tasks being scheduled at level 0 (highest priority) in TiFlash's unified read pool. Helps diagnose read request prioritization issues and identify when the system is consistently processing high-priority requests, which may indicate resource contention or workload imbalance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 25, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Vote", "group": "Raft message", "used_for": "**Used For:**\n\nUsed for monitoring the rate of Raft vote messages sent by TiFlash-Proxy instances during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 22, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel full events in TiFlash-Proxy, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, message processing bottlenecks, and potential data synchronization delays between TiFlash and TiKV.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 59, "dashboard": "DM-Professional-Clinic", "title": "dispatch binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of dispatching binlog events in DM (Data Migration) replication tasks at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when event dispatch delays are impacting replication lag between source and target databases.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 59, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read flow", "group": "Storage", "used_for": "Used for monitoring the read I/O throughput in TiFlash storage layer, including file descriptor reads, PageStorage (PSM) reads, and background reads. Essential for diagnosing read performance bottlenecks, identifying excessive disk I/O, and understanding storage access patterns during query execution.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4198, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiFlash-Proxy's unified read pool at any given time. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient thread resources or slow query processing.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Check split", "group": "Raft admin", "used_for": "Used for monitoring the rate of region split checks in TiFlash-Proxy's Raft store, categorized by check type. Helps diagnose region split behavior and identify potential issues with region management or split decision-making in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 76, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft admin", "used_for": "Used for monitoring the rate of Raft administrative operations (configuration changes and leader transfers) in TiFlash-Proxy. Essential for tracking cluster topology changes and diagnosing issues related to region leadership transfers or membership modifications that may impact TiFlash's data replication stability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiFlash-Proxy, broken down by instance and error reason. Critical for identifying issues with analytical query processing and diagnosing failures in the coprocessor request handling pipeline between TiDB and TiFlash.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft admin", "used_for": "Used for monitoring the rate of successful Raft admin commands in TiFlash-Proxy, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes that affect TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1715, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the number of Raft leader regions on each TiFlash node and detecting significant leader transfers. Helps identify load imbalance across TiFlash instances and diagnose unexpected leader migration events (drops of more than 10 leaders within 2 minutes), which may indicate node instability or cluster rebalancing issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the rate of different storage command types executed in TiFlash-Proxy. Helps identify command distribution patterns and diagnose issues when specific storage operations (like get, scan, prewrite) show abnormal rates or spikes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 75, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the number of TiFlash-Proxy raft regions actively receiving write operations per instance. Helps identify write hotspots and uneven write distribution across TiFlash nodes, which can indicate data skew or replication issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1705, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the total storage size of each TiFlash-Proxy instance's RocksDB engine. Essential for tracking disk space usage, capacity planning, and identifying instances approaching storage limits that may require scaling or data cleanup.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 21, "dashboard": "DM-Professional-Clinic", "title": "storage remain", "group": "relay log", "used_for": "**Used For:**\n\nUsed for monitoring the available disk space remaining for DM relay logs. Critical for preventing relay log write failures due to insufficient storage and ensuring continuous data replication from upstream MySQL to downstream TiDB.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"available\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 84, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Throughput", "group": "Storage", "used_for": "Used for monitoring the I/O throughput controlled by TiFlash's I/O limiter, broken down by operation type (read/write). Essential for identifying I/O bottlenecks and verifying whether storage operations are being throttled due to configured I/O limits.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 31, "dashboard": "DM-Professional-Clinic", "title": "relay log data corruption", "group": "relay log", "used_for": "Used for detecting data corruption events in DM (Data Migration) relay logs. Critical for identifying data integrity issues during binlog replication that could lead to inconsistent data migration or replication failures.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_data_corruption{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 132, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data Sharing", "group": "Storage", "used_for": "Used for monitoring TiFlash's data sharing cache mechanism, which allows multiple read threads to share data blocks in memory. Helps diagnose cache efficiency by tracking cache hit rates, miss rates, and cache operations, essential for optimizing TiFlash read performance and memory utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"add_cache_stale|add_cache_succ|add_cache_total_bytes_limit|add_cache_reach_count_limit\"}[2m])) by (type)", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))/sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 57, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region in TiFlash-Proxy. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 169, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Effectiveness of Mark Cache", "group": "Storage", "used_for": "Used for monitoring the hit and miss rates of TiFlash's mark cache, which stores metadata about data block boundaries. Essential for evaluating cache efficiency and diagnosing performance issues related to metadata lookups during query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_system_profile_event_MarkCacheMisses{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tiflash_system_profile_event_MarkCacheHits{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 102, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS", "group": "binlog replication", "used_for": "Used for monitoring the throughput of binlog replication operations in DM (Data Migration), measured as queries per second. Helps identify replication performance bottlenecks and track the rate of different operation types (INSERT/UPDATE/DELETE) being replicated from source to target databases.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval]))by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 1718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiFlash-Proxy's Raftstore engine, broken down by instance and error status. Critical for identifying storage engine errors and replication issues that could impact TiFlash's ability to sync data from TiKV.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3638, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiFlash-Proxy's Raft store. Helps identify region size imbalances and diagnose issues related to region splitting, merging, or uneven data distribution that could impact I/O performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3834, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the volume of data being written by the scheduler in TiFlash-Proxy instances. Helps identify write bottlenecks, uneven write distribution across instances, and potential memory pressure issues in the scheduler's write buffer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 8, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the error rate of asynchronous storage requests in TiFlash-Proxy's storage engine. Critical for identifying storage layer failures, request timeouts, or engine errors that could impact TiFlash query performance and data synchronization.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3636, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the rate of bytes written to regions in TiFlash-Proxy per second. Helps identify write-heavy workloads, diagnose I/O bottlenecks, and detect abnormal write patterns that may impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1481, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the distribution of Region sizes in TiFlash-Proxy's Raft store, showing p95, p99, and average values. Essential for identifying oversized Regions that may cause I/O bottlenecks, slow Raft replication, or unbalanced data distribution across TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and relay", "group": "relay log", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM's relay log. Helps identify replication lag at the file level and diagnose relay log synchronization delays or issues in data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"} - ON(instance) dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 15, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing slow snapshot generation issues that can impact TiFlash's ability to serve consistent reads and replicate data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 193, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands in TiFlash-Proxy instances. Helps identify scheduling bottlenecks and command queue buildup that could indicate resource contention or processing delays in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Snapshots", "group": "Storage", "used_for": "Used for monitoring the number of active read snapshots and segment read tasks in TiFlash storage engine. Essential for diagnosing MVCC snapshot accumulation issues, memory pressure from long-running queries, and identifying potential read workload bottlenecks that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "DM-Professional-Clinic", "title": "binlog file index", "group": "relay log", "used_for": "Used for tracking the current binlog file index position in DM's relay log processing, comparing the master's binlog file against the relay log's file position. Essential for monitoring replication lag and identifying if relay log processing is falling behind the upstream MySQL master.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 20, "dashboard": "DM-Professional-Clinic", "title": "storage capacity", "group": "relay log", "used_for": "Used for monitoring the total storage capacity allocated for DM (Data Migration) relay logs. Helps administrators track available disk space for relay log storage and plan capacity before running out of space during data replication tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"capacity\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 238, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Mark Cache and Minmax Index Cache Memory Usage", "group": "Storage", "used_for": "Used for monitoring memory consumption of TiFlash's data indexing structures, including mark cache (data location markers), minmax index files, and delta index cache. Essential for diagnosing TiFlash memory pressure issues and optimizing cache configurations when analytical query performance degrades.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_RNDeltaIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 291, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory by class", "group": "Storage", "used_for": "Used for monitoring TiFlash memory consumption broken down by internal memory allocation classes. Helps identify which components (e.g., cache, query execution, storage) are consuming memory and diagnose memory pressure or OOM issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 289, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory by thread", "group": "Storage", "used_for": "Used for monitoring memory allocation and deallocation across different threads in TiFlash storage engine. Helps identify memory leaks, excessive memory consumption by specific threads, and diagnose out-of-memory issues in TiFlash instances.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"alloc_.*\"}", "-tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dealloc_.*\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 167, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the total number of scheduler operations across different stages in TiFlash-Proxy, including both normal operations and \"too busy\" rejections. Helps diagnose scheduler bottlenecks and resource contention issues when TiFlash-Proxy cannot process requests due to overload conditions.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 6, "dashboard": "DM-Professional-Clinic", "title": "binlog pos", "group": "relay log", "used_for": "Used for monitoring the binlog position tracking in DM's relay log processing, comparing the master's binlog position against the relay log's current position. Essential for identifying replication lag, detecting relay log sync delays, and diagnosing data migration bottlenecks between upstream MySQL and DM relay components.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 266, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Pending Rate and Duration", "group": "Storage", "used_for": "Used for monitoring the rate of I/O operations waiting in TiFlash's I/O limiter queue, broken down by operation type. Essential for identifying I/O throttling issues and diagnosing storage performance bottlenecks when TiFlash experiences read/write delays due to rate limiting.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter_pending_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "DM-Professional-Clinic", "title": "write relay log duration", "group": "relay log", "used_for": "Used for monitoring the write latency of relay logs in DM (Data Migration) at different percentiles (p90, p95, p99). Critical for diagnosing relay log write performance issues and identifying bottlenecks in the data replication pipeline that could impact migration throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 109, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing TiFlash write performance bottlenecks and identifying when data ingestion from TiKV is experiencing delays or slowdowns.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 86, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Current Pending Count", "group": "Storage", "used_for": "Used for monitoring the number of write requests currently queued in TiFlash's I/O limiter, broken down by rate limiter, background writes, and foreground writes. Essential for diagnosing write throttling issues and identifying when storage I/O limits are causing request backlog in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 28, "dashboard": "DM-Professional-Clinic", "title": "fail to read binlog from master", "group": "relay log", "used_for": "Used for monitoring the count of errors encountered when DM (Data Migration) relay log fails to read binlog from the upstream MySQL master. Critical for diagnosing replication interruptions and connectivity issues between DM relay and the source database.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_read_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 95, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages processed by TiFlash-Proxy, broken down by message type. Helps identify communication patterns between TiFlash and TiKV, and diagnose issues with inter-component RPC traffic or message processing bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "DM-Professional-Clinic", "title": " transform binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of transforming binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when event transformation is causing replication lag or delays in the data migration pipeline.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 269, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Scheduling", "group": "Storage", "used_for": "Used for monitoring TiFlash read thread scheduling behavior and resource utilization patterns. Helps diagnose read performance bottlenecks by tracking thread exhaustion, active segment limits, cache hits, and scheduling efficiency in the storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 29, "dashboard": "DM-Professional-Clinic", "title": "fail to write relay log", "group": "relay log", "used_for": "Used for monitoring relay log write failures in DM (Data Migration) workers. Critical for diagnosing data replication interruptions caused by I/O errors when persisting binlog events to local relay logs.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_write_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 168, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiFlash-Proxy's transaction scheduler, including tracking when the scheduler becomes too busy to accept requests. Essential for diagnosing write command processing bottlenecks and identifying which scheduler stage (snapshot, async_apply_prewrite, etc.) is causing performance degradation or resource saturation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3637, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the rate of keys written to regions in TiFlash-Proxy's storage engine. Helps diagnose write throughput issues and identify regions experiencing heavy write activity that may impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 292, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Internal Duration", "group": "Storage", "used_for": "Used for monitoring the internal processing latency of TiFlash read threads at various percentiles (p80-p999). Critical for diagnosing TiFlash query performance bottlenecks and identifying whether read operations are experiencing delays in internal thread processing, helping distinguish between network, scheduling, and actual execution latency issues.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiFlash-Proxy. Helps identify priority distribution of I/O operations and diagnose whether high-priority commands are being properly processed or if low-priority commands are overwhelming the scheduler.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 168, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema of Column File", "group": "Storage", "used_for": "Used for monitoring TiFlash's column file schema cache performance, tracking the current cache size, hit rate, and eviction behavior. Essential for diagnosing schema cache efficiency issues and identifying when schema metadata is being evicted while still in use, which can impact query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)", "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[2m])) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 294, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Compression Ratio", "group": "Storage", "used_for": "Used for monitoring the compression efficiency of TiFlash's columnar storage, showing ratios for both LZ4 and lightweight compression algorithms. Essential for evaluating storage space utilization and identifying potential compression issues that could impact disk usage and I/O performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lz4_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lz4_compressed_bytes\"}[2m]))", "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lightweight_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lightweight_compressed_bytes\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Essential for diagnosing I/O performance issues and identifying commands that scan excessive keys, which may indicate inefficient queries or hotspot problems in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler commands in TiFlash-Proxy at p95 and p99 percentiles. Essential for diagnosing performance bottlenecks in TiFlash's transaction scheduler operations, helping identify when specific commands (like prewrite, commit) are experiencing delays that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 373, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose write amplification issues and identify commands that are writing unexpectedly large numbers of keys, which could impact TiFlash storage performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of latch waiting in TiFlash-Proxy's scheduler, tracking p95, p99, and average wait times for transaction conflicts. Critical for diagnosing write contention issues when multiple transactions compete for the same keys, helping identify hotspot problems and scheduling bottlenecks in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2532, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Average gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC messages between TiFlash and TiKV, broken down by message type. Essential for diagnosing communication bottlenecks and identifying which specific gRPC operations are experiencing delays in the TiFlash-TiKV interaction layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 62, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Stall Duration", "group": "Storage Write Stall", "used_for": "Used for monitoring the 99th percentile duration of write stalls in TiFlash's storage engine. Critical for diagnosing write performance degradation caused by background compaction, memory pressure, or disk I/O bottlenecks that temporarily block incoming writes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, instance))", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 25, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "relay log", "used_for": "Used for monitoring the size distribution of binlog events written to relay logs in DM (Data Migration), showing p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that could impact relay log performance and diagnose replication bottlenecks caused by oversized transactions or DDL operations.", "component": "", "phenomenon": "", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 107, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by message type. Essential for identifying communication issues between TiFlash and other cluster components, helping diagnose network problems or service unavailability.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2534, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages sent through gRPC in TiFlash-Proxy. Helps diagnose message batching efficiency and identify potential network communication issues when batch sizes are abnormally small (indicating poor batching) or large (indicating potential message accumulation).", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 293, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Compression Algorithm Count", "group": "Storage", "used_for": "Used for monitoring the distribution of compression algorithms used by TiFlash storage engine when compressing data packs. Helps identify compression strategy patterns and diagnose storage efficiency issues related to data compression choices.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_pack_compression_algorithm_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 98, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message handling in TiFlash-Proxy. Critical for identifying communication bottlenecks between TiFlash and TiKV, helping diagnose slow data replication or query execution issues caused by inter-component network delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 30, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "relay log", "used_for": "Used for monitoring the count of relay log process abnormal exits in DM (Data Migration). Critical for identifying relay log stability issues and process crashes that could interrupt data replication from MySQL binlog to downstream TiDB.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_exit_with_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 9, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "relay log", "used_for": "Used for monitoring the latency of reading binlog events from the relay log in DM (Data Migration), showing p90, p95, and p99 percentiles. Critical for diagnosing slow binlog processing issues and identifying performance bottlenecks in the relay log reading stage that could impact data replication lag.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 83, "dashboard": "DM-Professional-Clinic", "title": "number of workers in different state", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of DM (Data Migration) workers across different operational states in the cluster. Essential for diagnosing high availability issues, identifying stuck or failed workers, and ensuring proper worker state transitions during data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "count_values(\"state\", dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 137, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Total", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage I/O throughput, separating write/ingest operations from delta management operations (compaction, GC). Critical for diagnosing write stall issues by identifying whether bottlenecks stem from incoming writes or background delta management tasks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})", "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of incoming requests to TiFlash-Proxy, showing both average and p99 percentile values by request type. Helps identify request batching efficiency and detect anomalies where requests aren't being properly batched, which could impact TiFlash query performance and resource utilization.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 675, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations on the lock column family within TiFlash-Proxy's scheduler for specific commands. Helps diagnose lock-related scanning performance and identify potential lock contention or scanning bottlenecks in distributed transactions.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch ratio", "group": "Server", "used_for": "Used for monitoring the batch efficiency of requests processed by TiFlash-Proxy, showing both average and P99 batch ratios by request type. Essential for identifying request batching inefficiencies that could impact TiFlash I/O performance and diagnosing whether requests are being properly grouped for optimal processing.", "component": "TIFLASH", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 830, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the distribution of key-value scan operations in TiFlash-Proxy's scheduler for the default column family. Helps diagnose scan performance issues by breaking down scan operations by tag, useful when investigating slow queries or identifying inefficient scan patterns in TiFlash storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 829, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations on TiFlash-Proxy's write column family during scheduler command execution. Helps diagnose write-path performance issues by revealing scan patterns and potential bottlenecks in the storage layer's write operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 560, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiFlash-Proxy's scheduler, broken down by operation tags. Helps diagnose scan performance issues and understand the characteristics of different scan types (e.g., processed keys, skipped keys) for specific scheduler commands.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 105, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC polling threads in TiFlash-Proxy instances. Essential for diagnosing gRPC communication overhead and identifying instances with excessive CPU consumption in network I/O operations.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 89, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Throughput By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring write and ingest throughput (bytes per second) for each TiFlash instance. Essential for identifying write performance bottlenecks, uneven write distribution across instances, and diagnosing storage write stall issues in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write\"}[2m])) by (instance)", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 86, "dashboard": "DM-Professional-Clinic", "title": "number of dm-masters start leader components per minute", "group": "HA", "used_for": "Used for monitoring the frequency of DM-master leader component initialization events per minute. Helps diagnose high availability issues, frequent leader switches, or cluster instability that may cause service disruptions and increased latency in data migration tasks.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_master_start_leader_counter" ], "dashboard_file": "dm.json" }, { "panel_id": 81, "dashboard": "DM-Professional-Clinic", "title": "workers' state", "group": "HA", "used_for": "Used for monitoring the operational state of DM (Data Migration) workers in the cluster. Essential for tracking worker availability and diagnosing high availability issues when workers fail or become unavailable during data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (worker) (dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 90, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring the rate of write operations and storage commands per TiFlash instance. Helps identify write throughput imbalances across instances and diagnose storage write stall issues by tracking DMWriteBlock events and storage command counts.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 84, "dashboard": "DM-Professional-Clinic", "title": "number of pending shard ddl", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the count of pending DDL operations in DM (Data Migration) shard merge scenarios, categorized by task and type. Essential for tracking DDL coordination status across sharded tables and identifying potential DDL synchronization bottlenecks or stuck operations in high availability deployments.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (dm_master_ddl_state_number)" ], "dashboard_file": "dm.json" }, { "panel_id": 64, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's scheduler worker threads, which handle task scheduling and coordination. Essential for identifying scheduler bottlenecks, thread contention issues, or abnormal CPU consumption that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2533, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile and average batch sizes of gRPC requests and responses in TiFlash-Proxy. Essential for diagnosing gRPC communication efficiency and identifying potential batching issues that could impact TiFlash query performance and network utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Throughput", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage layer throughput, separating write/ingest operations from delta management operations (compaction, merge). Critical for diagnosing write stall issues by identifying whether bottlenecks stem from incoming writes or background delta management processes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[2m]))", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 82, "dashboard": "DM-Professional-Clinic", "title": "shard ddl error per minute", "group": "HA", "used_for": "Used for monitoring the rate of shard DDL errors per minute in DM (Data Migration) master, broken down by task and error type. Critical for identifying DDL synchronization failures in sharded table migration scenarios and diagnosing high availability issues in DM clusters.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (increase(dm_master_shard_ddl_error[1m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 36, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiFlash-Proxy, including sending, applying, and generating snapshots. Critical for diagnosing slow replica synchronization and identifying performance bottlenecks in TiFlash's Raft snapshot handling process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 71, "dashboard": "DM-Professional-Clinic", "title": "before any operate error", "group": "operate error", "used_for": "Used for monitoring errors that occur in DM (Data Migration) workers before any operation begins. Helps diagnose pre-operation validation failures or initialization issues that prevent data migration tasks from starting properly.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"BeforeAnyOp\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 35, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent by TiFlash-Proxy nodes. Essential for tracking data replication activity and diagnosing issues with region snapshot transfers, which can indicate cluster rebalancing, node recovery, or performance bottlenecks in TiFlash data synchronization.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 79, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's async apply threads, which handle Raft log application to the storage engine. Essential for diagnosing write performance bottlenecks and identifying whether apply threads are saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 61, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash's Raft store threads, which handle Raft consensus protocol operations for data replication. Essential for diagnosing performance bottlenecks in TiFlash's data synchronization layer and identifying instances with excessive Raft processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 85, "dashboard": "DM-Professional-Clinic", "title": "number of worker event error", "group": "HA", "used_for": "Used for monitoring the count of worker event errors in DM (Data Migration) Master, categorized by error type. Critical for identifying high availability issues and diagnosing worker coordination failures that could impact data migration reliability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (type) (dm_master_worker_event_error)" ], "dashboard_file": "dm.json" }, { "panel_id": 72, "dashboard": "DM-Professional-Clinic", "title": "source bound error", "group": "operate error", "used_for": "Used for monitoring source binding operation errors in DM (Data Migration) workers. Essential for diagnosing connection or configuration issues when DM workers attempt to bind to upstream MySQL/MariaDB data sources, helping identify source connectivity problems that block data migration tasks.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"SourceBound\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 38, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots by type (sending/receiving/applying) in TiFlash-Proxy. Helps diagnose snapshot transfer issues, replica synchronization problems, and identify excessive snapshot activity that may impact cluster performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 128, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Disk Usage", "group": "PageStorage", "used_for": "Used for monitoring disk space usage of TiFlash's PageStorage blob files, showing total disk bytes, valid data bytes, and space utilization ratio per instance. Essential for tracking storage efficiency, identifying space waste from deleted or obsolete data, and determining when garbage collection or disk expansion is needed.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})) by (instance)", "tiflash_system_asynchronous_metric_LogDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4287, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's unified read pool threads, which handle data read operations. Essential for identifying read performance bottlenecks and determining if the read pool is CPU-saturated during heavy query workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 133, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage WriteBatch Size", "group": "PageStorage", "used_for": "Used for monitoring the distribution of write batch sizes in TiFlash's PageStorage v3 engine. Helps diagnose write efficiency and identify whether data is being written in optimal batch sizes, which is critical for understanding I/O patterns and potential performance bottlenecks in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_page_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"v3\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1908, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash-Proxy's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether specific priority queues are saturated or experiencing CPU contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 129, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage File Num", "group": "PageStorage", "used_for": "Used for monitoring the number of blob files and log files in TiFlash's PageStorage engine per instance. Essential for tracking storage file accumulation and identifying potential issues with file compaction or cleanup processes that could impact storage efficiency and performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_BlobFileNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_asynchronous_metric_LogNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "DM-Professional-Clinic", "title": "start error", "group": "operate error", "used_for": "Used for monitoring start operation errors in DM (Data Migration) workers. Essential for diagnosing failures when DM workers attempt to start data migration tasks or subtasks, helping identify configuration issues or resource constraints preventing successful task initialization.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Start\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 74, "dashboard": "DM-Professional-Clinic", "title": "pause error", "group": "operate error", "used_for": "Used for monitoring the number of errors encountered when pausing DM (Data Migration) workers or tasks. Critical for diagnosing operational issues during pause operations and ensuring smooth task lifecycle management in DM clusters.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Pause\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 69, "dashboard": "DM-Professional-Clinic", "title": "auto-resume error", "group": "operate error", "used_for": "Used for monitoring auto-resume operation failures in DM (Data Migration) workers. Critical for identifying when automatic task recovery mechanisms fail, requiring manual intervention to restore data synchronization.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"AutoResume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 158, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page write Duration", "group": "PageStorage", "used_for": "Used for monitoring write latency percentiles (P80, P95, P99, P999) in TiFlash's PageStorage engine. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow page write operations that could impact columnar data ingestion and storage efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 69, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiFlash-Proxy instances. Essential for identifying CPU bottlenecks in the storage engine layer and diagnosing performance issues related to data compaction, writes, and reads in TiFlash's underlying RocksDB.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 163, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Tasks OPM", "group": "PageStorage", "used_for": "Used for monitoring the rate of PageStorage garbage collection tasks per minute in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential issues with storage space reclamation or excessive GC overhead that could impact query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 44, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of snapshot sizes in TiFlash-Proxy's Raft replication process. Critical for identifying abnormally large snapshots that could cause memory pressure, network congestion, or slow replica synchronization during TiFlash node recovery or scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 78, "dashboard": "DM-Professional-Clinic", "title": "resume error", "group": "operate error", "used_for": "Used for monitoring the error count when resuming DM (Data Migration) tasks or subtasks. Essential for diagnosing issues with task recovery operations and identifying problems that prevent successful task resumption after pauses or failures.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Resume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 39, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of small internal storage tasks in TiFlash, excluding major operations like delta merge, segment merge, and segment split. Helps identify the activity level of background maintenance tasks and diagnose potential bottlenecks in TiFlash's internal storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 43, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pair counts in TiFlash-Proxy snapshots. Helps identify extreme cases where snapshots contain unusually large numbers of KV pairs, which can indicate data skew issues or potential performance bottlenecks during snapshot processing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1395, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy worker queues, broken down by worker name. Essential for identifying task queue bottlenecks and worker thread saturation that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1876, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiFlash-Proxy's future pool threads, broken down by pool name. Helps identify task processing throughput and potential bottlenecks in TiFlash-Proxy's asynchronous task execution system.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 162, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Duration", "group": "PageStorage", "used_for": "Used for monitoring the 99th percentile latency of TiFlash PageStorage garbage collection operations. Essential for diagnosing performance degradation in TiFlash's storage layer, particularly when GC pauses impact query response times or cause storage management bottlenecks.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 67, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy background worker threads across instances. Essential for identifying background task overhead and diagnosing performance issues caused by maintenance operations like compaction, cleanup, or internal housekeeping tasks.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 198, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage stored bytes by type", "group": "PageStorage", "used_for": "Used for monitoring the distribution of stored data bytes across different types in TiFlash's PageStorage engine. Helps identify storage composition, detect abnormal growth in specific data types, and diagnose storage space usage issues.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_page_data_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2531, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's garbage collection worker threads. Essential for diagnosing GC performance issues and identifying instances where excessive GC activity may be impacting overall TiFlash performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 75, "dashboard": "DM-Professional-Clinic", "title": "stop error", "group": "operate error", "used_for": "Used for monitoring the frequency of stop operation errors in DM (Data Migration) workers. Essential for identifying issues when attempting to stop data migration tasks or worker processes, helping diagnose operational failures and cluster management problems.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Stop\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 164, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Numer of Pages", "group": "PageStorage", "used_for": "Used for monitoring the number of pages stored in TiFlash's PageStorage engine, tracking both in-memory pages and versioned entries. Essential for understanding TiFlash memory usage patterns and diagnosing issues related to page storage capacity or memory pressure in the columnar storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_PagesInMem{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_VersionedEntries{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1877, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy's future pool, broken down by pool name. Helps identify task queue buildup and potential thread pool saturation issues that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 78, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash coprocessor threads across different priority levels (normal, high, and low). Essential for diagnosing TiFlash query processing bottlenecks and identifying instances with excessive CPU consumption during analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 68, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read Index Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's Read Index worker threads, which handle Raft read index requests to ensure read consistency. Essential for diagnosing performance bottlenecks when TiFlash queries experience high latency due to read index processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 120, "dashboard": "DM-Professional-Clinic", "title": "Validator Error Count", "group": "Validator Statistics", "used_for": "Used for monitoring the total number of validation errors encountered by DM's data validator component. Critical for identifying data consistency issues during migration and triggering investigation when validation failures occur between upstream and downstream data.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_error_count" ], "dashboard_file": "dm.json" }, { "panel_id": 59, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiFlash-Proxy. Helps identify worker thread activity patterns and diagnose task processing bottlenecks or imbalances across worker pools.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of threads in TiFlash-Proxy instances, showing how many threads are in different states (running, idle, blocked, etc.) per instance. Essential for diagnosing thread pool saturation, identifying I/O bottlenecks, and detecting thread starvation issues that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 114, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog file position being processed by the DM (Data Migration) validator. Helps track validation progress and identify if the validator is lagging behind the source database's binlog replication stream.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_file" ], "dashboard_file": "dm.json" }, { "panel_id": 116, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the data replication lag in bytes between the DM validator and the source binlog position. Critical for identifying data synchronization delays and ensuring the validator is keeping pace with upstream changes during data migration validation.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logpos_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 231, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Pending Writers Num", "group": "PageStorage", "used_for": "Used for monitoring the number of pending write operations in TiFlash's PageStorage component. Helps identify write bottlenecks and I/O congestion when pending writers accumulate, indicating potential disk performance issues or write pressure on TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_PSPendingWriterNum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1069, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of PD (Placement Driver) requests sent from TiFlash-Proxy instances. Helps identify communication patterns between TiFlash-Proxy and PD, and diagnose issues related to metadata operations, region information queries, or TSO (timestamp oracle) requests.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 670, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region task worker pre-handle/generate snapshot CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's region task and worker threads responsible for pre-handling requests and generating snapshots. Critical for diagnosing TiFlash region replication bottlenecks and identifying instances with excessive CPU consumption during snapshot generation or region task processing.", "component": "TIFLASH", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 35, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Read Index OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations per second in TiFlash instances. Helps identify read throughput patterns and diagnose issues when TiFlash queries require strong consistency reads through the Raft protocol.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 77, "dashboard": "DM-Professional-Clinic", "title": "update error", "group": "operate error", "used_for": "Used for monitoring the rate of update operation errors in DM (Data Migration) workers. Helps identify issues with metadata updates or configuration changes that may be failing, indicating potential problems with DM's operational stability or connectivity to PD.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Update\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 232, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PS Command OPS By Instance", "group": "PageStorage", "used_for": "Used for monitoring the operations per second (OPS) of PageStorage commands in TiFlash, broken down by instance and command type. Essential for identifying which TiFlash instances are handling the most PageStorage operations and detecting imbalances or performance bottlenecks in the storage layer.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_page_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring TiProxy server CPU utilization and comparing it against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation, and determining if TiProxy needs vertical scaling or load distribution.", "component": "TIPROXY", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}[2m])", "tiproxy_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 1215, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages exchanged between TiFlash-Proxy and PD (Placement Driver). Essential for diagnosing communication issues between TiFlash-Proxy and PD, which can indicate network problems or PD overload affecting cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 138, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation efficiency in TiFlash-Proxy, tracking hit rates across different storage layers (memtable, block cache, and L0 SST files). Essential for diagnosing read performance issues and understanding how effectively data is being served from memory versus disk in the storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 123, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Number of Tables", "group": "PageStorage", "used_for": "Used for monitoring the number of tables stored in TiFlash's PageStorage across different storage pool modes (V2, V3, and mixed mode). Helps track table distribution across TiFlash instances and identify storage mode transitions or imbalances in table placement.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 6, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiProxy instances. Essential for tracking connection pool utilization, identifying connection leaks, and diagnosing capacity issues when connection counts approach limits or show unexpected spikes.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 1396, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation requests between TiFlash-Proxy and PD (Placement Driver). Helps diagnose issues with region peer metadata synchronization and identify potential problems in TiFlash-Proxy's communication with PD for peer state verification.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2258, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes per second) across different TiFlash-Proxy threads, filtered to show only threads with I/O rates exceeding 1KB/s. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or imbalanced workload distribution in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2660, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring the rate of voluntary context switches in TiFlash-Proxy threads, indicating when threads willingly yield CPU due to I/O waits or resource blocking. High rates (>200/sec) help diagnose I/O bottlenecks, lock contention, or threads frequently waiting for external resources, which can impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 200" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 167, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Stale Read OPS", "group": "Raft", "used_for": "Used for monitoring the rate of stale read operations per second on each TiFlash instance. Helps identify stale read traffic patterns and diagnose issues when applications are using stale read features to query historical data from TiFlash replicas.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_stale_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 112, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog position (in bytes) being processed by the DM validator component. Helps track data validation progress and identify potential lag or stalls in the validation process during data migration tasks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_pos" ], "dashboard_file": "dm.json" }, { "panel_id": 271, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Index Events", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations in TiFlash, categorized by event type. Essential for diagnosing read consistency issues and understanding the frequency of linearizable read requests that require Raft consensus verification.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_read_index_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 270, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Learner Read Failures", "group": "Raft", "used_for": "Used for monitoring the rate of learner read failures in TiFlash, categorized by failure type. Critical for diagnosing data replication issues between TiKV and TiFlash, helping identify when TiFlash cannot successfully read data from Raft learner replicas, which may impact analytical query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_learner_read_failures_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1070, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different types of requests from TiFlash-Proxy to PD (Placement Driver). Critical for diagnosing communication delays between TiFlash compute nodes and PD, which can impact metadata operations and region information updates.", "component": "PD", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 125, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiFlash-Proxy, tracking maximum, P99, and P95 durations. Essential for diagnosing read performance issues and identifying slow iterator operations when TiFlash queries experience delays accessing underlying storage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 7, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Create Connection OPM", "group": "Server", "used_for": "Used for monitoring the rate of new connections created per minute on each TiProxy instance. Helps identify connection creation patterns, detect sudden spikes in client connection requests, and diagnose potential connection pool issues or client behavior anomalies.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_create_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 118, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the latency of validator binlog processing from file in DM (Data Migration). Helps diagnose delays in data validation when reading and processing binlog events from local files, which is critical for identifying bottlenecks in the validation pipeline.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logfile_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 82, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB point lookup (Get operation) latency in TiFlash-Proxy, showing maximum, P99, and P95 durations in microseconds. Essential for diagnosing read performance issues and identifying slow Get operations that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiProxy instances. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact proxy performance and stability.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiproxy\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 2661, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches in TiFlash-Proxy threads, which occur when the OS forcibly preempts threads due to CPU contention or time slice expiration. High rates (>50/sec) indicate CPU saturation or resource contention issues that can degrade TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 50" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 36, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Batch Read Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of batch read index operations in TiFlash's Raft layer at different percentiles (p80, p95, p99). Critical for diagnosing slow read performance issues and identifying when TiFlash instances experience delays in obtaining read indexes from the Raft consensus protocol, which can impact query response times.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (decode)", "group": "Raft", "used_for": "Used for monitoring the time spent decoding Raft write data in TiFlash regions. Critical for diagnosing performance bottlenecks in TiFlash's data ingestion pipeline when replicating data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 37, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Wait Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash waiting for Raft log index synchronization at p80, p95, and p99 percentiles. Critical for diagnosing data replication delays between TiKV and TiFlash, helping identify when TiFlash replicas fall behind in applying Raft logs, which can impact query freshness and read performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 139, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiFlash-Proxy's RocksDB storage engine, including successful writes (self/other), write timeouts, and WAL writes. Essential for diagnosing TiFlash write performance issues, identifying write bottlenecks, and detecting timeout anomalies that may indicate storage layer problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiProxy's memory consumption, including both resident memory (RSS) and Go heap memory usage. Essential for detecting memory leaks, identifying out-of-memory risks, and understanding TiProxy's resource footprint during connection pooling and query routing operations.", "component": "TIPROXY", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 242, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Heavy Raft Apply Duration", "group": "Raft", "used_for": "Used for monitoring the 99th percentile latency of Raft command apply operations in TiFlash. Critical for diagnosing replication lag and data synchronization performance issues between TiKV and TiFlash, especially when queries on TiFlash show stale data or high latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 126, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write operation latency in TiFlash-Proxy's RocksDB storage engine, showing maximum, 99th percentile, and 95th percentile write durations. Essential for diagnosing TiFlash write performance bottlenecks and identifying when storage layer operations are causing delays in data synchronization from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 82, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration", "group": "Raft", "used_for": "Used for monitoring the p99 latency of applying Raft write logs in TiFlash's storage engine. Critical for diagnosing write performance bottlenecks and replication lag issues when TiFlash processes data changes from TiKV through the Raft protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m]))", "sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m]) ) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 130, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) operations in TiFlash-Proxy's RocksDB storage engine, tracking maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues that affect TiFlash data persistence and durability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 137, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations in TiFlash-Proxy's RocksDB engine. Critical for diagnosing write durability overhead and identifying potential I/O bottlenecks when WAL sync frequency becomes excessive or irregular.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 12, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency through TiProxy at the 95th and 99th percentiles. Essential for identifying performance degradation in client-to-TiDB connections and diagnosing whether slowdowns originate from the proxy layer or backend TiDB servers.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 10, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Owner", "group": "Server", "used_for": "Used for identifying which TiProxy instance is currently serving as the owner/leader in the cluster. Essential for troubleshooting leadership changes, failover events, and understanding which instance is actively managing cluster coordination.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_owner{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 8, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Disconnection OPM", "group": "Server", "used_for": "Used for monitoring the rate of client disconnections per minute in TiProxy, categorized by disconnection type. Essential for identifying connection stability issues, detecting abnormal disconnect patterns, and diagnosing network or client-side problems affecting database connectivity.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 128, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiFlash-Proxy, broken down by operation type. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification patterns that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 135, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) file synchronization operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing write performance bottlenecks and disk I/O issues, as high WAL sync duration directly impacts transaction commit latency and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 129, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring RocksDB seek operation rates in TiFlash-Proxy, tracking total seeks, successful seeks, and next operations. Essential for diagnosing read performance issues and understanding data access patterns in TiFlash's storage layer, particularly when investigating slow queries or inefficient range scans.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 14, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Backend", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency from TiProxy to each backend TiDB instance. Essential for identifying which specific backend servers are experiencing slow query performance and diagnosing uneven load distribution or backend-specific issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, backend)), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 241, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Frequent Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of frequent Raft consensus events in TiFlash, broken down by event type. Essential for diagnosing Raft replication activity, identifying abnormal consensus patterns, and troubleshooting data synchronization issues between TiFlash replicas.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_frequent_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1708, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring the CPU utilization rate of TiFlash-Proxy processes across the cluster. Essential for identifying CPU bottlenecks, resource contention, and determining if TiFlash-Proxy nodes are overloaded or require scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 235, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft consensus events per second in TiFlash nodes, broken down by event type. Essential for diagnosing Raft replication issues, identifying abnormal consensus activity, and troubleshooting data synchronization problems between TiFlash replicas.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 15, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Instance", "group": "Query Summary", "used_for": "Used for monitoring the number of commands per second (CPS) processed by each TiProxy instance. Helps identify load distribution across TiProxy instances and detect performance bottlenecks or uneven traffic routing.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 93, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft admin logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft admin commands in TiFlash through a heatmap visualization. Essential for diagnosing performance issues related to administrative operations like region splits, merges, and configuration changes in TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 16, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Backend", "group": "Query Summary", "used_for": "Used for monitoring the number of queries per second (CPS) handled by each TiDB backend instance through TiProxy. Helps identify load distribution imbalances across backend servers and detect if specific TiDB instances are receiving disproportionate traffic or experiencing connection issues.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 13, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Instance", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency for each TiProxy instance. Essential for identifying performance degradation, comparing latency across instances, and diagnosing which specific TiProxy nodes are experiencing slow query issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 136, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiFlash-Proxy, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage performance issues, as prolonged compaction times can indicate write amplification problems or resource bottlenecks affecting query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 243, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Entry Batch Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft log entry batch sizes in TiFlash across different size buckets. Helps identify abnormal Raft replication patterns, such as unusually large batches that could indicate replication bottlenecks or memory pressure issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_entry_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"normal\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 88, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the memtable hit rate in TiFlash-Proxy's RocksDB storage engine. Helps diagnose memory efficiency and read performance by showing how often data is found in memory (memtable) versus requiring disk access, with higher hit rates indicating better memory utilization.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 87, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (write blocks)", "group": "Raft", "used_for": "Used for monitoring the latency distribution of TiFlash region write operations when persisting data blocks to storage. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow disk I/O issues that impact data replication from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Backend Connections", "group": "Balance", "used_for": "Used for monitoring the number of active backend connections from TiProxy to each TiDB instance. Essential for verifying connection load balancing across TiDB nodes and identifying connection distribution imbalances that could lead to uneven resource utilization.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_balance_b_conn{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 249, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ongoing raft snapshot", "group": "Raft", "used_for": "Used for monitoring the rate of ongoing Raft snapshot data transfer in TiFlash, broken down by snapshot type (apply/generate). Essential for diagnosing region replication issues, identifying snapshot bottlenecks, and understanding data synchronization overhead between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiFlash-Proxy's RocksDB engine, tracking maximum, P99, and P95 latencies when writes are blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when TiFlash-Proxy experiences backpressure that could impact data replication from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 17, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) in TiProxy, broken down by command type (SELECT, INSERT, UPDATE, etc.). Helps identify query workload patterns and detect unusual spikes in specific command types that may indicate application behavior changes or potential issues.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cmd_type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 234, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Log Gap Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft log gaps between applied_index and compact_index in TiFlash replicas. Helps identify replication lag issues and diagnose when TiFlash is falling behind in applying or compacting Raft logs, which can impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"applied_index\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"compact_index\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 18, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Handshake Duration", "group": "Query Summary", "used_for": "Used for monitoring the 95th and 99th percentile latency of client connection handshakes through TiProxy. Critical for diagnosing slow connection establishment issues and identifying authentication or SSL/TLS negotiation bottlenecks that impact client connection times.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_handshake_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_handshake_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 103, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables in TiFlash-Proxy, broken down by column family. Essential for diagnosing memory pressure issues and understanding write buffer usage that can impact write performance and memory stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 102, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the top 20 block cache sizes across different column families in TiFlash-Proxy's RocksDB engine. Essential for tracking memory usage patterns and identifying potential cache pressure or memory allocation issues that could impact read performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 140, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file read operations in TiFlash-Proxy's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage layer performance issues, particularly when queries experience slow data retrieval from disk or when investigating I/O bottlenecks in the columnar storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 80, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the block cache hit ratio in TiFlash-Proxy's RocksDB storage engine, including overall cache hits, data block hits, and filter block hits. Essential for evaluating memory efficiency and identifying when insufficient cache size causes excessive disk I/O, which directly impacts TiFlash query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 81, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft write log commands in TiFlash nodes. Essential for diagnosing write performance bottlenecks and identifying when TiFlash is experiencing delays in processing replicated data from TiKV through the Raft consensus protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 240, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft eager garbage collection operations in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential issues with Raft log cleanup that could impact memory usage and storage efficiency.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_eager_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 261, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Committed Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of committed write sizes in TiFlash's Raft layer over time. Helps identify write patterns, detect abnormal write size distributions, and diagnose I/O performance issues related to data replication in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write_committed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 265, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ingest Uncommitted Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of uncommitted data sizes during TiFlash's Raft ingestion process as a heatmap. Helps identify data ingestion patterns and potential memory pressure issues when uncommitted data accumulates excessively before being committed to storage.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest_uncommitted\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 468, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache operations in TiFlash-Proxy's RocksDB storage engine, including overall block additions, data block additions, and filter block additions. Essential for understanding cache population patterns and diagnosing I/O performance issues related to cache management efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 132, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiFlash-Proxy's RocksDB storage engine, including keys read, keys written, and corrupt keys detected during compaction. Essential for diagnosing data flow issues, identifying unusual read/write patterns, and detecting potential data corruption problems in TiFlash storage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 29, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of data received by TiProxy from backend TiDB servers, measured in bytes per second. Helps identify backend-to-proxy traffic patterns, detect network bottlenecks, and diagnose imbalanced data distribution across TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 244, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Size (by event) Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of region data sizes during TiFlash's Raft flush operations, showing both flushed and unflushed data volumes as a heatmap. Helps identify region size patterns and diagnose I/O performance issues by revealing whether regions are accumulating too much unflushed data or experiencing irregular flush behavior.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"unflushed\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"flushed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 21, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration OPM", "group": "Balance", "used_for": "Used for monitoring the rate of session migrations per minute in TiProxy's load balancing system, showing migration flows between backend TiDB instances and their success/failure status. Essential for diagnosing connection rebalancing behavior, identifying migration bottlenecks, and troubleshooting session redistribution issues during scaling or failover events.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(label_replace(sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (migrate_res, from, to), \"from\", \"$1\", \"from\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\"), \"to\", \"$1\", \"to\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 26, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Ping Backend Duration", "group": "Backend", "used_for": "Used for monitoring the network latency between TiProxy and backend TiDB servers through health check pings. Essential for diagnosing connection quality issues and identifying network problems that could affect query routing performance.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_backend_ping_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 30, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets received by TiProxy from backend TiDB servers. Helps identify network throughput patterns, backend communication issues, and potential packet loss or congestion between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 131, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys in each column family of TiFlash-Proxy's RocksDB storage engine. Helps assess data volume distribution across column families and identify potential storage imbalances or unexpected data growth in TiFlash's storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 85, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of TiFlash-Proxy's RocksDB storage engine, tracking both direct bytes read and iterator-based reads. Essential for diagnosing TiFlash read performance issues, identifying I/O bottlenecks, and understanding data access patterns during analytical query execution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 134, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiFlash-Proxy's RocksDB storage engine. Helps diagnose write amplification issues and identify abnormally large write operations that could impact I/O performance and storage efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 467, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write traffic flow and data insertion rate of TiFlash-Proxy's RocksDB block cache. Essential for analyzing cache utilization patterns and diagnosing memory pressure or cache efficiency issues in TiFlash storage engine.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 5, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiProxy instance has been running since its last restart. Essential for tracking instance stability, identifying unexpected restarts, and verifying deployment or maintenance activities.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 133, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data read per RocksDB read operation in TiFlash-Proxy, showing maximum, P99, and P95 values. Helps diagnose read amplification issues and identify whether queries are reading excessive data, which can indicate inefficient storage layout or suboptimal query patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 25, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Get Backend Duration", "group": "Backend", "used_for": "Used for monitoring the latency of TiProxy selecting and connecting to backend TiDB instances, showing p95, p99, and average durations. Critical for diagnosing connection establishment delays and backend selection performance issues that could impact client query response times.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_backend_get_backend_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_backend_get_backend_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 22, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Duration", "group": "Balance", "used_for": "Used for monitoring the latency of session migration operations in TiProxy during load balancing, tracking p95 and p99 percentiles. Critical for diagnosing performance issues when TiProxy redistributes client connections across TiDB instances, helping identify slow migrations that could impact user experience during rebalancing events.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_balance_migrate_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_balance_migrate_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 75, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Applying snapshots Count", "group": "Raft", "used_for": "Used for monitoring the snapshot application process in TiFlash's Raft layer, tracking pending snapshots, prehandling subtasks, and parallel prehandling tasks. Critical for diagnosing region replication delays, snapshot processing bottlenecks, and identifying instances struggling with Raft snapshot synchronization during data recovery or scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumPrehandlingSubTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumParallelPrehandlingTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 27, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Health Check Cycle", "group": "Backend", "used_for": "Used for monitoring the health check cycle duration of backend TiDB servers in TiProxy. Essential for diagnosing connection pool health and detecting delays in backend availability detection that could impact connection routing and failover responsiveness.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_backend_health_check_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 262, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Uncommitted Size Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of uncommitted snapshot data sizes in TiFlash's Raft replication layer. Helps identify memory pressure issues and diagnose problems with snapshot application delays or excessive uncommitted data accumulation during Raft log replication.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"snapshot_uncommitted\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 72, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding Raft snapshots during data replication. Critical for diagnosing slow snapshot application issues and identifying bottlenecks in TiFlash's Raft log replay process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 272, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory by thread", "group": "Raft", "used_for": "Used for monitoring memory allocation and deallocation across different Raft proxy threads in TiFlash. Helps identify memory leaks, excessive memory consumption by specific threads, and diagnose memory-related performance issues in TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"alloc_.*\"}", "-tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dealloc_.*\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 23, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Reasons", "group": "Balance", "used_for": "Used for tracking the reasons why TiProxy migrates client sessions between TiDB instances during load balancing operations. Essential for understanding session migration patterns and diagnosing whether migrations are triggered by planned rebalancing, instance failures, or configuration changes.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 86, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring write throughput in TiFlash-Proxy's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written. Essential for diagnosing write performance bottlenecks, identifying unusual write patterns, and understanding I/O load on TiFlash-Proxy storage during data replication and persistence operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 31, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the outbound network traffic rate from TiProxy instances to backend TiDB servers. Helps identify network bandwidth usage patterns, detect traffic imbalances across backends, and diagnose connectivity or throughput issues between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 32, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets sent from TiProxy instances to backend TiDB servers. Helps identify network traffic distribution patterns, detect connection issues to specific backends, and diagnose packet-level communication problems between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 239, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash's Raft eager garbage collection operations at p95 and p99 percentiles. Essential for diagnosing performance issues related to Raft log cleanup and identifying when eager GC operations are taking longer than expected, which could impact TiFlash's replication efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_eager_gc_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_raft_eager_gc_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 90, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the data flow rates during RocksDB compaction operations in TiFlash-Proxy, including bytes read, bytes written during compaction, and flush write bytes. Essential for diagnosing I/O bottlenecks, understanding compaction overhead, and identifying excessive write amplification that may impact TiFlash storage performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 33, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Cross Location Bytes/Second", "group": "Traffic", "used_for": "Used for monitoring the rate of cross-location network traffic in bytes per second handled by TiProxy instances. Essential for identifying cross-region data transfer costs and diagnosing network bottlenecks when TiDB clusters span multiple availability zones or geographic locations.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_traffic_cross_location_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 127, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode SST to DT Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding SST files to DeltaTree format during Raft snapshot application. Critical for diagnosing slow snapshot ingestion and identifying performance bottlenecks when TiFlash replicas are catching up with data changes from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode_sst2dt\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 55, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Capacity size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity configured for each TiFlash instance. Essential for capacity planning and identifying instances approaching storage limits before they impact cluster operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 263, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "KVStore memory", "group": "Raft", "used_for": "Used for monitoring the memory consumption of TiFlash's KVStore component, which handles Raft-based data replication and storage. Essential for identifying memory pressure issues in TiFlash's storage layer and diagnosing potential out-of-memory problems during data synchronization.", "component": "TIFLASH", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingKVStore{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 290, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Prehandle Throughput Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft snapshot prehandling throughput across TiFlash instances as a heatmap. Helps identify performance bottlenecks and uneven workload distribution during Raft snapshot processing, which is critical for diagnosing data replication delays and cluster synchronization issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_throughput_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"prehandle_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 260, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft throughput", "group": "Raft", "used_for": "Used for monitoring the Raft log replication throughput in TiFlash nodes, broken down by operation type. Essential for diagnosing data synchronization performance between TiKV and TiFlash, and identifying bottlenecks in the Raft replication pipeline that may cause TiFlash replicas to lag behind.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 516, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiFlash-Proxy's storage engine. Helps identify potential memory leaks or snapshot accumulation issues that can impact TiFlash-Proxy performance and resource usage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 71, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Keys flow", "group": "Raft", "used_for": "Used for monitoring the rate of Raft log keys processed by TiFlash, categorized by operation type (such as apply, snapshot). Essential for tracking data replication throughput from TiKV to TiFlash and diagnosing replication lag or bottlenecks in the Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 250, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft snapshot sizes in TiFlash over time as a heatmap. Helps identify abnormal snapshot size patterns, detect large snapshot transfers that may impact network bandwidth, and diagnose replication performance issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_snapshot_total_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"approx_raft_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 518, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiFlash-Proxy's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O during read operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 863, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression ratio across different LSM-tree levels in TiFlash-Proxy's RocksDB storage engine. Helps identify compression efficiency issues and storage space optimization opportunities, particularly when investigating unexpected disk usage growth or I/O performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 200, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region", "group": "Server", "used_for": "Used for monitoring the total number of Raft regions stored on each TiFlash instance. Essential for tracking data distribution across TiFlash nodes and identifying region imbalance issues that could impact query performance and storage utilization.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 73, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Flush Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash takes to flush Raft snapshots to disk. Critical for diagnosing slow snapshot application issues that can cause TiFlash replicas to lag behind TiKV, impacting data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 517, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiFlash-Proxy's RocksDB engine. Critical for identifying long-running transactions or stale snapshots that can block compaction, cause storage bloat, and degrade read/write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2003, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST file ingestion operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing snapshot application performance issues and identifying bottlenecks during data replication from TiKV to TiFlash.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 127, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the volume of data pending compaction in TiFlash-Proxy's RocksDB storage engine, broken down by column family. Critical for identifying compaction backlogs that can lead to read amplification, increased disk usage, and degraded I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2002, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in TiFlash-Proxy's RocksDB storage engine for each column family. Essential for diagnosing compaction issues, storage amplification problems, and identifying when LSM-tree structure becomes unbalanced due to write pressure or compaction bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 21, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiFlash instance has been running since its last restart. Helps quickly identify recent restarts, instance stability issues, and verify cluster continuity after maintenance or unexpected failures.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 53, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the storage space consumed by each TiFlash instance, including both local storage (StoreSizeUsed) and remote storage (StoreSizeUsedRemote). Essential for tracking disk usage trends, capacity planning, and identifying instances approaching storage limits that may require scaling or data cleanup.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"\"}) by (instance)", "sum(tiflash_system_current_metric_StoreSizeUsedRemote{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 54, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Available size", "group": "Server", "used_for": "Used for monitoring the available disk space on each TiFlash instance. Critical for preventing storage exhaustion issues and planning capacity expansion before TiFlash nodes run out of disk space.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2381, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency of stall condition changes in TiFlash-Proxy's RocksDB column families. Critical for diagnosing write throttling and I/O pressure issues that cause RocksDB to enter stall states, which can severely impact write performance and cause request timeouts.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 51, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiFlash instances, showing both actual CPU usage and available logical CPU cores. Essential for identifying CPU bottlenecks, resource saturation, and determining if TiFlash nodes need vertical scaling or load rebalancing.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])", "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 92, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log replication latency from TiKV to TiFlash instances over time. Essential for diagnosing data synchronization delays and identifying network or replication bottlenecks between TiKV and TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 74, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ingest SST Duration", "group": "Raft", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations in TiFlash's Raft layer. Critical for diagnosing performance issues during data replication from TiKV to TiFlash, especially when experiencing slow data synchronization or snapshot application delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 264, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Big Write To Region Size Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of large write operations to TiFlash regions over time. Helps identify write hotspots and diagnose uneven data distribution or excessive large writes that may impact TiFlash performance and stability.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"big_write_to_region\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2452, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons why RocksDB write operations are being stalled in TiFlash-Proxy. Essential for diagnosing write performance bottlenecks by identifying whether stalls are caused by memtable limits, L0 file accumulation, pending compaction bytes, or other RocksDB flow control mechanisms.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 10, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory", "group": "Server", "used_for": "Used for monitoring TiFlash memory allocation through jemalloc metrics, including retained, mapped, and resident memory across the cluster. Essential for diagnosing memory pressure issues, detecting memory leaks, and understanding TiFlash's actual memory footprint versus allocated memory.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_MemoryCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 181, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "IO Throughput", "group": "Server", "used_for": "Used for monitoring the I/O throughput rate (bytes per second) across TiFlash instances. Essential for identifying I/O bottlenecks, detecting abnormal disk activity, and diagnosing performance issues related to data read/write operations in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 91, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency", "group": "Raft", "used_for": "Used for monitoring the latency of Raft log replication from TiKV to TiFlash at p95, p99, and maximum percentiles. Critical for diagnosing data synchronization delays between TiKV and TiFlash, helping identify network issues or upstream bottlenecks that could cause stale reads in TiFlash replicas.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 99, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the effectiveness of TiFlash's rough set filter in eliminating unnecessary data reads during query execution. Helps identify whether the rough set index is successfully filtering out data blocks, which is critical for diagnosing query performance issues and understanding TiFlash's data pruning efficiency.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (instance)", "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 141, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Import Service", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST import service threads, which handle importing SST files during data replication and recovery. Essential for diagnosing performance bottlenecks during data ingestion, region migration, or when TiFlash replicas are being synchronized from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"sst_importer.*\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 58, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region in TiFlash-Proxy instances. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2451, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of RocksDB compaction triggers in TiFlash-Proxy by column family and reason type. Helps diagnose storage layer performance issues by identifying whether compactions are triggered by size thresholds, file counts, or manual operations, which is critical for understanding write amplification and I/O patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4492, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiFlash-Proxy's storage layer. Helps verify encryption-at-rest is properly enabled and track the volume of encrypted data files across the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4453, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiFlash-Proxy's encryption key management system. Essential for tracking encryption key lifecycle and diagnosing issues with data-at-rest encryption configuration or key rotation problems.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 286, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "In-Memory Vector Index Instances", "group": "Vector Search", "used_for": "Used for monitoring the number of active vector index instances loaded in TiFlash memory, broken down by type and instance. Essential for tracking memory resource usage of vector search features and identifying potential memory pressure from vector index operations.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (type, instance) (\n tiflash_vector_index_active_instances{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\"\n }\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 97, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate Histogram", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the distribution of rough set filter effectiveness in TiFlash's storage layer. Helps identify how efficiently TiFlash is filtering data blocks during queries, with higher filter rates indicating better query performance through reduced data scanning.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4497, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the size of encryption metadata files in TiFlash-Proxy. Helps track storage overhead from encryption key management and identify potential issues with metadata file growth that could impact disk usage.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 147, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash region worker threads, which handle region data synchronization from TiKV. Essential for diagnosing performance bottlenecks in TiFlash's data replication process and identifying when region workers are CPU-saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 154, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Apply", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST apply threads, which handle low-priority Raft log applications to the storage engine. Essential for diagnosing write performance bottlenecks and identifying when apply threads are saturated during heavy write workloads or data replication.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 288, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Vector Index Estimated Memory Usage", "group": "Vector Search", "used_for": "Used for monitoring the estimated memory consumption of vector indexes in TiFlash, including both in-memory index structures and file-backed memory mappings. Critical for capacity planning and diagnosing out-of-memory issues when running vector search workloads.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_vector_index_memory_usage{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\"\n}", "tiflash_process_rss_by_type_bytes{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\",\n type=\"file\"\n}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 282, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "99.9% Vector Search Duration (Per Request)", "group": "Vector Search", "used_for": "Used for monitoring the 99.9th percentile latency of vector search operations in TiFlash. Critical for identifying tail latency issues in vector similarity searches and diagnosing performance degradation that affects the slowest vector queries.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(\n 0.999, \n sum(rate(\n tiflash_vector_index_duration_bucket{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\",\n type!=\"build\"\n }\n [$__rate_interval]\n )) by (le, type)\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 155, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Store", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of Raft Store threads in TiFlash's proxy layer, which handle Raft consensus operations and data replication. Essential for diagnosing performance bottlenecks in TiFlash's distributed consensus mechanism and identifying instances with high replication overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 145, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Task", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash region task threads, which handle region-related operations like region split, merge, and metadata updates. Essential for diagnosing performance bottlenecks in region management and identifying instances with abnormal region processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 284, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "99.9% Vector Index Build Duration (Per DMFile Column)", "group": "Vector Search", "used_for": "Used for monitoring the 99.9th percentile latency of vector index building operations per DMFile column in TiFlash. Critical for identifying performance bottlenecks and outliers when building vector indexes, which directly impacts vector search query preparation time and overall system responsiveness.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(\n 0.999, \n sum(rate(\n tiflash_vector_index_duration_bucket{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\",\n type=\"build\"\n }\n [$__rate_interval]\n )) by (le, type)\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk process duration", "group": "", "used_for": "Used for monitoring the average time taken to process each data chunk during TiDB Lightning import operations. Essential for identifying import performance bottlenecks and estimating overall data migration completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "1/rate(lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 151, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Small Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background worker threads that handle small maintenance tasks like compaction and cleanup. Helps identify resource contention or performance bottlenecks in background operations that could impact overall TiFlash storage efficiency.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4496, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether encryption-at-rest has been successfully initialized in TiFlash-Proxy instances. Essential for confirming encryption setup during cluster deployment or when troubleshooting encryption-related configuration issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1, "dashboard": "Lightning-Cluster-Clinic", "title": "Import speed", "group": "", "used_for": "Used for monitoring the data import throughput during TiDB Lightning operations, tracking write speed to TiKV, upload speed, and overall imported bytes per second. Essential for diagnosing import performance bottlenecks and ensuring data migration tasks are progressing at expected rates.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"imported\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"restored\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"written\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 173, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency breakdown of read operations in TiFlash's disaggregated storage architecture. Helps identify which component (compute or storage layer) is contributing most to read latency, essential for diagnosing performance bottlenecks in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 257, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's apply worker threads, which handle Raft log application and data replication from TiKV. Critical for diagnosing replication lag issues and identifying CPU bottlenecks in TiFlash's data synchronization process.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 8, "dashboard": "Lightning-Cluster-Clinic", "title": "Failures", "group": "", "used_for": "Used for monitoring the number of table import failures in TiDB Lightning across the cluster. Essential for quickly identifying data import issues and tracking the reliability of bulk data loading operations.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", result=\"failure\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 156, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Large Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background threads handling large tasks like compaction and data reorganization. Essential for diagnosing performance bottlenecks when background operations consume excessive CPU resources, impacting query performance or causing resource contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 159, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Server", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async server threads that handle asynchronous network operations. Critical for identifying whether async I/O processing is becoming a bottleneck and if thread pool resources need adjustment.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4498, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the latency of reading and writing encryption metadata files in TiFlash-Proxy, showing p95 and maximum durations. Critical for diagnosing performance issues with encryption-at-rest operations and identifying when metadata I/O becomes a bottleneck affecting TiFlash query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4, "dashboard": "Lightning-Cluster-Clinic", "title": "Import Progress", "group": "", "used_for": "Used for monitoring the progress of data import operations in TiDB Lightning by tracking the ratio of finished chunks to estimated total chunks. Essential for estimating import completion time and identifying stalled or slow import jobs that may require intervention.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"estimated\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 196, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload keys speed by type (all)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint key uploads in TiFlash's disaggregated storage architecture, broken down by key type. Helps diagnose checkpoint performance issues and identify which key types are contributing most to checkpoint upload activity in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_keys_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 176, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote File Num", "group": "Disaggregated", "used_for": "Used for monitoring the number of remote storage files in TiFlash's disaggregated storage architecture. Helps track remote file accumulation and identify potential issues with file management or storage tiering in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"num_files\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 149, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Manual Compaction", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's manual compaction thread pool. Helps identify when manual compaction operations are consuming excessive CPU resources, which may indicate heavy background maintenance activity or potential performance bottlenecks in storage layer optimization.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "Lightning-Cluster-Clinic", "title": "Checksum progress", "group": "", "used_for": "Used for tracking the completion progress of table checksum verification during TiDB Lightning data import. Helps monitor how many tables have completed checksum validation versus the total number of tables, essential for ensuring data integrity and estimating remaining import time.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"pending\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 175, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Store Usage", "group": "Disaggregated", "used_for": "Used for monitoring remote storage usage in disaggregated TiFlash deployments, tracking total size, valid data size, and storage efficiency ratio per instance. Essential for managing cloud storage costs and identifying data bloat or inefficient space utilization in TiFlash's remote storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"}) by (instance)", "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) by (instance)", "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"})) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 153, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Sender", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's snapshot sender threads, which handle sending Raft snapshots to other replicas. Critical for diagnosing performance issues during region replication, data recovery, or when adding new TiFlash nodes to the cluster.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 187, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload Duration", "group": "Disaggregated", "used_for": "Used for monitoring the latency of checkpoint upload operations in TiFlash disaggregated storage architecture at different percentiles (p80, p90, p99). Critical for diagnosing performance issues with TiFlash's checkpoint synchronization to remote storage, which can impact data durability and recovery capabilities in disaggregated deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 161, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Client", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async client threads that handle remote procedure calls. Essential for diagnosing communication bottlenecks between TiFlash and other TiDB components, especially when experiencing high network request latency or thread saturation issues.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 174, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow", "group": "Disaggregated", "used_for": "Used for monitoring the data upload flow rate to remote storage in TiFlash disaggregated architecture, tracking both incremental checkpoint uploads and compaction-related uploads. Essential for diagnosing write-to-S3 performance issues and understanding checkpoint synchronization efficiency in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"incremental\"}[2m]))", "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"compaction\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "Lightning-Cluster-Clinic", "title": "Memory usage", "group": "", "used_for": "Used for monitoring memory consumption of TiDB Lightning and its importer component during data import operations. Critical for detecting memory leaks, preventing OOM issues, and ensuring sufficient resources are allocated for large-scale data migration tasks.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 295, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Local Index Pool", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's local index thread pool, which handles index-related operations. Essential for diagnosing performance bottlenecks in index processing and determining if the local index pool requires tuning when query performance degrades.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\"})", "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexSched*\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 256, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FAP builder", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's FAP (Fast Add Peer) builder threads, which handle peer replication and region snapshot building. Critical for diagnosing performance bottlenecks during region replication, data migration, or when adding new TiFlash replicas to the cluster.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"fap_builder.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 197, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow by type (incremental+compaction)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint data upload flow in TiFlash's disaggregated storage architecture, broken down by operation type (incremental updates and compaction). Essential for diagnosing checkpoint upload performance, identifying bottlenecks in data persistence, and understanding the balance between incremental and compaction workloads in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3, "dashboard": "Lightning-Cluster-Clinic", "title": "CPU%", "group": "", "used_for": "Used for monitoring CPU utilization percentage of TiDB Lightning and Importer processes during data import operations. Essential for identifying resource bottlenecks, ensuring Lightning has sufficient CPU capacity for optimal import performance, and detecting abnormal CPU spikes that may indicate processing issues.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}[2m])*100", "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}[2m])*100" ], "dashboard_file": "lightning.json" }, { "panel_id": 9, "dashboard": "Lightning-Cluster-Clinic", "title": "Number of Lightning Goroutines", "group": "", "used_for": "Used for monitoring the number of active goroutines in TiDB Lightning instances during data import operations. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact import performance or stability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 189, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Request QPS", "group": "Disaggregated", "used_for": "Used for monitoring the rate of remote object lock requests per second in TiFlash's disaggregated storage architecture, broken down by request type. Essential for diagnosing lock contention issues and understanding coordination overhead between compute and storage nodes in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_disaggregated_object_lock_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Status", "group": "Disaggregated", "used_for": "Used for monitoring the status of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for tracking S3-based storage cleanup activities and diagnosing issues with stale data removal in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_s3_gc_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance,type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 21, "dashboard": "Lightning-Cluster-Clinic", "title": "Import/Normal mode", "group": "", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration across instances during Lightning import operations. Helps identify whether TiKV is in import mode (higher limit) or normal mode (lower limit), which is critical for diagnosing write performance issues and understanding cluster configuration state during bulk data imports.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "lightning.json" }, { "panel_id": 267, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Scheduler", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's segment scheduler threads, which manage data segment operations like merging and splitting. Essential for diagnosing performance bottlenecks in TiFlash's background maintenance tasks and identifying when segment scheduling becomes a resource constraint.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 193, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the P90 and P99 latency breakdown of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for diagnosing performance bottlenecks in S3-based storage cleanup processes and identifying which GC operation types are experiencing high latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 5, "dashboard": "Lightning-Cluster-Clinic", "title": "Idle workers", "group": "", "used_for": "Used for monitoring the number of idle worker threads in TiDB Lightning during data import operations. Helps identify resource utilization efficiency and potential bottlenecks—too many idle workers may indicate I/O constraints or upstream data supply issues, while consistently zero idle workers suggests the import is CPU/worker-bound.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_idle_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 268, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Reader", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SegmentReader threads, which handle reading data from storage segments during query execution. Essential for identifying compute bottlenecks in TiFlash's columnar storage layer and diagnosing performance issues when scanning large datasets.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 191, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Duration", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency of remote object lock requests in TiFlash's disaggregated storage architecture. Critical for diagnosing performance bottlenecks in distributed object storage access and identifying contention issues in the disaggregated compute-storage separation model.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 201, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Unit", "group": "Disaggregated", "used_for": "Used for monitoring Request Unit (RU) consumption in TiFlash's disaggregated storage and compute architecture. Helps track resource usage across keyspaces and compute clusters, essential for capacity planning and identifying abnormal resource consumption patterns in serverless or multi-tenant deployments.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)", "sum(increase(tiflash_storage_sync_replica_ru[24h])) by (keyspace_id)", "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)", "sum(increase(tiflash_compute_request_unit[24h])) by (cluster_id)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Duration", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash coprocessor request latency at multiple percentiles (p80, p95, p99, p999). Essential for diagnosing slow analytical query performance and identifying tail latency issues in TiFlash's columnar storage engine when handling pushed-down coprocessor requests from TiDB.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 6, "dashboard": "Lightning-Cluster-Clinic", "title": "External resources", "group": "", "used_for": "Used for monitoring the number of active external resources (KV encoders, importer engines, and open engine RPC calls) currently in use by TiDB Lightning during data import operations. Helps identify resource leaks or accumulation issues when resources are opened but not properly closed, which can lead to memory exhaustion or import failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 19, "dashboard": "Lightning-Cluster-Clinic", "title": "SST process duration", "group": "", "used_for": "Used for monitoring the performance of SST (Sorted String Table) file processing during TiDB Lightning data import operations. Helps diagnose import bottlenecks by tracking SST chunk throughput, split duration, and upload latency to TiKV nodes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 2, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Executor QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second handled by different TiFlash coprocessor executors, broken down by executor type (e.g., table scan, aggregation, join). Essential for understanding TiFlash query workload distribution and identifying which executor operations are most frequently used or experiencing throughput changes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 13, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk parser read block duration", "group": "", "used_for": "Used for monitoring the average duration of chunk parser read block operations and I/O apply worker latency in TiDB Lightning during data import. Critical for diagnosing data ingestion performance bottlenecks and identifying slow file reading or parsing issues that may impact overall import speed.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_chunk_parser_read_block_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_chunk_parser_read_block_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_apply_worker_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) /rate(lightning_apply_worker_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 233, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory Usage of Storage Tasks", "group": "Disaggregated", "used_for": "Used for monitoring memory consumption of storage-related tasks in TiFlash's disaggregated storage architecture, including query storage tasks, page fetching operations, and delta index cache. Critical for diagnosing memory pressure issues and optimizing resource allocation in TiFlash's storage layer, especially when investigating OOM errors or performance degradation in disaggregated mode.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DT_DeltaIndexCacheSize{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingSharedColumnData{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 9, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second (QPS) of coprocessor requests handled by TiFlash, broken down by request type. Essential for tracking TiFlash's analytical query workload and identifying sudden changes in request patterns or throughput bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "Lightning-Cluster-Clinic", "title": "Total bytes", "group": "", "used_for": "Used for monitoring data volume metrics during TiDB Lightning import operations, including bytes read from source data, bytes delivered to TiKV, and total cluster storage size. Essential for tracking import progress, estimating completion time, and ensuring sufficient storage capacity during large-scale data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_row_read_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"max_replicas\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 236, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexCache", "group": "Disaggregated", "used_for": "Used for monitoring the delta index cache hit rate and access patterns in TiFlash's disaggregated storage architecture. Essential for evaluating cache efficiency and diagnosing performance issues related to delta index lookups, where low hit rates may indicate insufficient cache size or suboptimal data access patterns.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, instance)", "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit\"}[2m])) by (instance) /sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 16, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process rate", "group": "", "used_for": "Used for monitoring the throughput rate of SQL data processing in TiDB Lightning during data import operations. Helps identify import performance bottlenecks and track the speed at which Lightning delivers data blocks to the target cluster.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "lightning.json" }, { "panel_id": 15, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process duration", "group": "", "used_for": "Used for monitoring the latency of SQL processing stages during Lightning data import, specifically tracking row encoding duration and block delivery duration. Essential for diagnosing performance bottlenecks in the Lightning import pipeline and identifying whether slowdowns occur during data encoding or delivery phases.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 252, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FAP state", "group": "Disaggregated", "used_for": "Used for monitoring the state distribution of FAP (Fast Analyze Process) tasks in TiFlash's disaggregated architecture. Helps track task execution patterns and identify potential bottlenecks or failures in the FAP task processing pipeline.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_fap_task_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 63, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Cop task memory usage", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring memory consumption of coprocessor requests in TiFlash at various percentiles (p80-p999). Essential for identifying memory-intensive queries, detecting potential OOM risks, and optimizing TiFlash resource allocation when coprocessor tasks consume excessive memory.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 14, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Response Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the rate of response data bytes per second returned by TiFlash coprocessor requests, broken down by request type. Helps identify data transfer bottlenecks and diagnose network bandwidth issues when TiFlash returns large result sets to TiDB.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Error QPS", "group": "Coprocessor", "used_for": "Used for monitoring the rate of coprocessor request errors per second in TiFlash, broken down by error reason. Critical for identifying and diagnosing issues with query execution failures, helping pinpoint whether errors stem from schema mismatches, region unavailability, or other coprocessor-level problems.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 13, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Handle Duration", "group": "Coprocessor", "used_for": "Used for monitoring the tail latency of coprocessor request processing in TiFlash at multiple percentiles (p80, p95, p99, p999). Essential for identifying slow coprocessor requests and diagnosing TiFlash query performance issues, particularly when analytical queries experience unexpected delays or timeouts.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 165, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Exchange Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the data exchange rate (bytes per second) between TiFlash nodes during distributed query execution, broken down by exchange type. Essential for diagnosing network bottlenecks and data shuffle performance issues in MPP queries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_exchange_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 253, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FAP no match reason", "group": "Disaggregated", "used_for": "Used for tracking reasons why TiFlash's Fast Aggregation Pushdown (FAP) optimization cannot be applied to queries in disaggregated mode. Helps identify query patterns or conditions preventing FAP optimization, enabling performance tuning and understanding why certain queries don't benefit from pushdown acceleration.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_fap_nomatch_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\n" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 251, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FAP result", "group": "Disaggregated", "used_for": "Used for monitoring the result distribution of FAP (Fast Aggregation Pushdown) tasks in TiFlash's disaggregated architecture. Helps identify task execution patterns and diagnose issues with pushdown computation success rates by tracking different result types (success, failure, timeout, etc.).", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_fap_task_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 100, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the current number of active RPC threads in TiFlash instances handling coprocessor requests. Essential for diagnosing thread pool saturation, identifying instances under heavy RPC load, and troubleshooting coprocessor request processing bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 178, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Bytes", "group": "S3", "used_for": "Used for monitoring S3 storage I/O throughput in TiFlash, tracking read/write bytes per second including DMFile operations. Essential for diagnosing S3 storage performance bottlenecks and understanding data transfer patterns when TiFlash uses S3 as the storage backend.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3WriteDMFileBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 254, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FAP time by stage", "group": "Disaggregated", "used_for": "Used for monitoring the 99.9th percentile execution time of different stages in TiFlash's FAP (Fast Analytical Processing) tasks in disaggregated mode. Critical for identifying performance bottlenecks in specific FAP processing stages and diagnosing slow analytical query issues in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(round(1000000000*rate(tiflash_fap_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 104, "dashboard": "DM-Professional-Clinic", "title": "Task State", "group": "", "used_for": "Used for monitoring the current state of DM (Data Migration) worker tasks across different sources and jobs. Essential for tracking task health, identifying stuck or failed migration tasks, and ensuring data replication is running smoothly.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 87, "dashboard": "DM-Professional-Clinic", "title": "load progress", "group": "dump and load ", "used_for": "Used for monitoring the data loading progress in DM's lightning component by tracking the ratio of finished chunks to estimated total chunks. Essential for understanding how far along the data migration load phase has progressed and estimating remaining time for completion.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"} / ignoring(state) lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"estimated\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 18, "dashboard": "Lightning-Cluster-Clinic", "title": "Deliver duration", "group": "", "used_for": "Used for monitoring the latency of data delivery operations during TiDB Lightning import processes. Critical for diagnosing bottlenecks in the data ingestion pipeline, including range delivery to TiKV, SST file delivery, and row-based KV pair delivery from Lightning to the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_row_kv_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_kv_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 95, "dashboard": "DM-Professional-Clinic", "title": "checksum progress", "group": "dump and load ", "used_for": "Used for monitoring the checksum verification progress during DM's data migration process. Helps track the ratio of completed table checksums to pending checksums, indicating how close the data validation phase is to completion.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"completed\"} / ignoring(state) lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"pending\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 88, "dashboard": "DM-Professional-Clinic", "title": "dump progress", "group": "dump and load ", "used_for": "Used for monitoring the data dump progress in DM (Data Migration) tasks by showing the ratio of finished rows to estimated total rows. Essential for tracking dump phase completion and estimating remaining time during full data migration from source databases.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dumpling_dump_finished_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"} / dumpling_dump_estimate_total_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 77, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Handling Request Number", "group": "Coprocessor", "used_for": "Used for monitoring the number of coprocessor requests currently being processed by TiFlash instances, categorized by request type. Helps identify request processing bottlenecks and diagnose issues when TiFlash is handling high concurrent workloads or experiencing request queuing.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 101, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the maximum number of RPC threads in TiFlash instances. Helps identify thread pool saturation and potential bottlenecks in coprocessor request handling that could impact query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 182, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Retry OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operation retries in TiFlash, including GetObject, PutObject, and PutDMFile operations. Essential for identifying S3 connectivity issues, network instability, or storage service degradation that causes TiFlash to retry operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFileRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 157, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "MPP Query count", "group": "Coprocessor", "used_for": "Used for monitoring the count of MPP (Massively Parallel Processing) queries being executed in TiFlash instances. Helps identify query distribution across TiFlash nodes and diagnose workload imbalances or query processing bottlenecks in analytical workloads.", "component": "TIFLASH", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 180, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Request Duration", "group": "S3", "used_for": "Used for monitoring S3 request latency in TiFlash at different percentiles (p80, p90, p99, p100). Critical for diagnosing performance issues when TiFlash uses S3-compatible object storage, helping identify slow S3 operations that may impact query performance or data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 185, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Operations", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache performance when using S3-based disaggregated storage, tracking cache operation rates and hit ratios for both DTFile and page-level caches. Essential for diagnosing I/O efficiency issues and optimizing cache effectiveness when TiFlash reads data from remote S3 storage.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit|dtfile_miss\"}[2m]))", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit|page_miss\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 179, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operations (PUT, GET, HEAD) performed by TiFlash instances when using S3-compatible storage. Essential for diagnosing S3 storage throughput issues, identifying unusual access patterns, and troubleshooting performance problems related to remote storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 102, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads", "group": "Coprocessor", "used_for": "Used for monitoring the current thread count across different thread pools in TiFlash instances, excluding RPC and maximum limit threads. Essential for identifying thread resource exhaustion, thread pool saturation, or abnormal thread creation that could impact TiFlash coprocessor query processing performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 33, "dashboard": "DM-Professional-Clinic", "title": "load process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's load process during data migration. Helps identify and diagnose failures in the data loading phase by tracking the rate of abnormal process terminations over 30-minute intervals.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_loader_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 32, "dashboard": "DM-Professional-Clinic", "title": "dump process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's dump process over 30-minute intervals. Critical for identifying dump task failures and data migration interruptions that require immediate attention.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_dumpling_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" } ]