[ { "panel_id": 8, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Size", "group": "Volume", "used_for": "Used for monitoring the requested storage capacity of PersistentVolumeClaims in the specified namespace. Helps track storage resource allocation and identify when volumes may need resizing or when storage quotas are being approached.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_resource_requests_storage_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 104, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write IO bytes", "group": "IO Breakdown", "used_for": "**Used For:**\n\nUsed for monitoring the rate of write I/O throughput (bytes per second) across TiKV instances. Essential for identifying write-heavy workloads, diagnosing disk I/O bottlenecks, and correlating storage layer performance with application write operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 316, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Total Encode/Deliver/Import-kv Speed", "group": "Import Into", "used_for": "Used for monitoring the throughput of TiDB's IMPORT INTO operation across three stages: encode (restored), deliver (written), and import-kv (imported). Essential for diagnosing data import performance bottlenecks and identifying which stage is limiting overall import speed.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"merged\"}[2m])) by(task_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 106, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IO threshold", "group": "IO Breakdown", "used_for": "Used for monitoring the configured I/O rate limits (throttling thresholds) in TiKV's rate limiter by operation type. Helps administrators verify I/O throttling settings and diagnose whether performance issues are caused by hitting rate limit constraints.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dataplane Resource Overview", "title": "Pod Status", "group": "Pod", "used_for": "Used for monitoring the current status phase of pods in the specified namespace. Helps quickly identify pod health issues by showing which pods are in Running, Pending, Failed, or other phases, enabling rapid detection of deployment or scheduling problems.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 320, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Import/Normal Mode", "group": "Import Into", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration during data import operations. Helps verify whether TiKV is in import mode (higher limit) or normal mode (standard limit), which is critical for diagnosing import performance issues and ensuring proper mode transitions.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 324, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Coprocessor Response Size Per Instance And TiKV", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor response data size from TiKV stores to each TiDB instance when using follower read. Helps identify network bandwidth usage patterns and diagnose issues where specific TiKV stores are sending disproportionate amounts of data, which may indicate load imbalance or inefficient query distribution in follower read scenarios.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 115, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store send proposal duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft proposals from TiKV store threads to Raft workers, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when the proposal submission phase becomes a performance bottleneck in the Raft consensus process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1309, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft message", "used_for": "Used for monitoring the rate of dropped Raft consensus messages in TiFlash-Proxy, categorized by message type. Critical for diagnosing Raft communication failures, network issues, or overload conditions that could lead to replication lag or cluster instability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 319, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Data Read/Encode/Deliver Average Duration", "group": "Import Into", "used_for": "Used for monitoring the average latency of three key stages in TiDB's IMPORT INTO operation: reading source data, encoding rows, and delivering blocks to storage. Essential for diagnosing performance bottlenecks during bulk data import tasks and identifying which stage is causing slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id) / sum(rate(tidb_import_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id)", "sum(rate(tidb_import_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id) / sum(rate(tidb_import_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id)", "sum(rate(tidb_import_row_read_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id) / sum(rate(tidb_import_row_read_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by(task_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 7, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft propose", "used_for": "Used for monitoring the rate of different types of Raft proposals in TiFlash-Proxy, including local reads, normal writes, read index requests, and batch operations. Essential for diagnosing I/O performance issues and understanding the distribution of read/write workload patterns in TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 24, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Request", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU resource requests and limits configured for each pod in the dataplane namespace. Essential for capacity planning, identifying pods with insufficient resource allocations, and detecting potential CPU throttling issues before they impact performance.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 317, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Encoded/Delivered/Imported Data Size", "group": "Import Into", "used_for": "Used for monitoring the data flow progress during IMPORT INTO operations in TiDB, tracking three key stages: encoded data size (total_restore), delivered data size (written), and successfully imported data size (restored) per task. Essential for diagnosing import performance bottlenecks, verifying data transfer completeness, and identifying tasks with encoding or delivery delays.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}) by(task_id)", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}) by(task_id)", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"total_restore\"}) by(task_id)", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}) by(task_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 22, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolumeClaim", "group": "Volume", "used_for": "Used for displaying information about PersistentVolumeClaims (PVCs) in the specified Kubernetes namespace. Helps administrators track storage volume claims, verify their configuration, and troubleshoot storage provisioning issues in the dataplane.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 116, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write kv db end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the duration of completing KV database writes in TiKV's raftstore, tracking p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when data persistence to RocksDB becomes a performance issue in the Raft replication waterfall.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 293, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "EMA CPU Usage", "group": "Resource Manager", "used_for": "Used for monitoring the Exponential Moving Average (EMA) of CPU usage by TiDB's Resource Manager. Helps track smoothed CPU consumption trends to identify resource allocation patterns and potential resource group throttling issues.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 101, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "KvEngine worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's transaction chunk workers that handle key-value engine operations. Essential for identifying CPU bottlenecks in transaction processing and diagnosing performance degradation when worker threads are overloaded or underutilized.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 109, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine, tracking both P99 and P9999 percentiles. Critical for diagnosing write performance bottlenecks and identifying tail latency issues in the storage layer that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft process", "used_for": "Used for monitoring the rate of Raft ready operations handled by TiFlash-Proxy, broken down by operation type. Essential for diagnosing Raft consensus processing performance and identifying bottlenecks in TiFlash's replication pipeline when data synchronization delays occur.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 6, "dashboard": "Dataplane Resource Overview", "title": "Deployment", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for each deployment in the specified namespace. Helps verify deployment scaling configurations and identify discrepancies between desired and actual replica counts during capacity planning or troubleshooting pod availability issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_deployment_spec_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 10, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Requests", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring memory resource requests and limits configured for pods in the dataplane namespace. Helps identify pod memory allocation settings and detect potential resource constraint issues when actual usage approaches or exceeds configured requests/limits.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 52, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Current Usage", "group": "Volume", "used_for": "Used for monitoring the current storage consumption of each PersistentVolume in the dataplane namespace. Helps identify volumes approaching capacity limits and diagnose storage-related issues before they impact cluster operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim) - sum(kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3690, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of committing Raft logs to disk in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing write performance bottlenecks and disk I/O issues in TiFlash's Raft replication layer, especially when data synchronization becomes slow.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 310, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Closest Replica Hit Count Per Second", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor requests served by the closest replica (follower) per second in TiDB's follower read feature. Helps verify follower read effectiveness and diagnose whether queries are successfully utilizing nearby replicas to reduce cross-AZ latency and improve read performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_closest_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 120, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiFlash node. Helps identify write workload distribution across TiFlash instances and diagnose imbalanced write traffic or nodes experiencing proposal bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 114, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store send to write queue duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft store thread messages to the write queue in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when the Raft store thread experiences delays in dispatching work to the write worker pool.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 289, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "GOGC", "group": "Resource Manager", "used_for": "Used for monitoring the GOGC (Go Garbage Collection) parameter value in TiDB instances. Helps diagnose memory management behavior and tune garbage collection frequency when investigating memory pressure or GC-related performance issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 25, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Vote", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft vote messages sent by TiFlash-Proxy instances during leader election processes. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 313, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Event Count Per Minute", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) events occurring in TiDB per minute, broken down by event type. Essential for tracking TTL job activity, identifying abnormal deletion patterns, and diagnosing issues with automatic data expiration and cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"hook.tidb.ttl\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"runtime\\\\.ttl.*\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 4, "dashboard": "Dataplane Resource Overview", "title": "StatefulSet", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for StatefulSets in the selected namespace. Helps verify StatefulSet scaling configurations and detect mismatches between desired and actual replica counts during deployment or scaling operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 281, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Job Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) jobs across different statuses in TiDB. Helps identify job execution patterns, detect stuck or failed TTL jobs, and assess the overall health of automatic data expiration processes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 67, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb)", "group": "HPA Overview", "used_for": "Used for monitoring Horizontal Pod Autoscaler (HPA) status for TiDB instances, showing desired vs. current replica counts and maximum scaling limits. Essential for verifying auto-scaling behavior and identifying when TiDB pods are scaling up/down in response to load changes or hitting configured capacity limits.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 108, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile of Raft proposals processed per ready batch in TiFlash-Proxy. Helps diagnose Raft replication efficiency and identify instances where proposal batching may be suboptimal, potentially indicating I/O bottlenecks or consensus performance issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 14, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Usage", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of individual pods within the selected namespace. Helps identify pods with high CPU consumption, detect resource bottlenecks, and diagnose performance issues or resource contention at the pod level.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (cluster,namespace,pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\", container!=\"POD\",container!=\"\"}[$__rate_interval])) " ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 99, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's Distributed File System (DFS) worker threads, including builtin-dfs and memory-related workers. Essential for diagnosing performance bottlenecks in TiKV's internal file operations and identifying when DFS threads are consuming excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"builtin-dfs.*|memory-fs.*|s3-client.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 113, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store batch wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft store threads spend waiting in batch queues before processing, tracking both p99 and p9999 percentiles. Critical for diagnosing Raft processing bottlenecks and identifying when store threads are blocked waiting for batch operations, which directly impacts write latency and replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "Dataplane Resource Overview", "title": "CPU Utilization", "group": "Pod", "used_for": "Used for monitoring CPU utilization of pods in the dataplane, showing both actual usage against limits and resource requests against node allocatable capacity. Essential for identifying CPU bottlenecks, over-provisioned pods, and node resource saturation issues.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)/\nsum(kube_pod_container_resource_limit_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\"}[$__rate_interval]\n ) and on (container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)\n/\nsum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)", "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 20, "dashboard": "Dataplane Resource Overview", "title": "DaemonSet", "group": "Pod", "used_for": "Used for monitoring the number of pods currently scheduled for each DaemonSet in the selected namespace. Helps verify that DaemonSets are properly deployed across nodes and identify scheduling issues when the actual pod count doesn't match expectations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_daemonset_status_current_number_scheduled{cluster=\"$cluster\",namespace=\"$namespace\"}) by (daemonset)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 11, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Messages", "group": "Raft message", "used_for": "Used for monitoring the rate of different types of Raft messages sent by TiFlash-Proxy nodes. Essential for diagnosing Raft communication issues, identifying message flooding, and understanding consensus protocol behavior in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 100, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Cloud worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's cloud worker threads that handle background tasks and asynchronous operations. Essential for identifying CPU bottlenecks in worker thread pools and diagnosing performance issues related to background job processing in cloud-native TiKV deployments.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker-server.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Delivered KV Count", "group": "Import Into", "used_for": "Used for monitoring the total number of key-value pairs delivered during IMPORT INTO operations in TiDB. Helps track data ingestion progress and diagnose slow import performance by showing KV delivery rates across different import tasks and data types.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_import_block_deliver_kv_pairs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(kind, task_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 32, "dashboard": "Dataplane Resource Overview", "title": "Docker Images", "group": "Image Versions", "used_for": "Used for monitoring the container images deployed across pods in the dataplane namespace. Helps track image versions, identify version inconsistencies, and verify successful deployments or rollouts across the cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_pod_container_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 112, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store propose wait duration", "group": "Raft Waterfall", "used_for": "**Used For:**\n\nUsed for monitoring the wait time before Raft proposals are processed in TiKV's raftstore thread, tracking p99 and p9999 latencies. Critical for diagnosing write path bottlenecks caused by raftstore thread saturation or queueing delays before proposals enter the Raft consensus process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 102, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Busy Threads (>80%)", "group": "Thread CPU", "used_for": "Used for identifying the top 20 TiKV threads consuming more than 80% CPU utilization. Critical for diagnosing CPU bottlenecks, detecting thread contention issues, and pinpointing which specific TiKV components (RocksDB, gRPC, scheduler, etc.) are causing high CPU load.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3688, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations in TiFlash-Proxy per server instance. Critical for diagnosing write path performance issues and identifying slow disk I/O or Raft replication bottlenecks that could impact TiFlash data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 304, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Insert/Delete Rows By Day", "group": "TTL", "used_for": "Used for monitoring the daily volume of TTL (Time To Live) insert and delete operations in TiDB. Helps track TTL job effectiveness and identify anomalies in automatic data expiration patterns, such as unexpected spikes in expired row deletions or insert activity.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 294, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Task Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) tasks across different statuses in TiDB. Helps track task execution states (running, completed, failed) to identify TTL job bottlenecks or failures that may prevent timely data cleanup.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 118, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Process ready duration per server", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile duration of Raft ready state processing in TiFlash-Proxy per server. Critical for diagnosing Raft replication delays and identifying which TiFlash nodes are experiencing slow consensus processing that could impact data synchronization performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 105, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of read I/O throughput in bytes per second across TiKV instances. Essential for diagnosing read performance bottlenecks, identifying instances with abnormal disk read activity, and correlating storage I/O patterns with query performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations in TiFlash-Proxy per server instance. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow Raft log persistence, which can impact data replication and consensus operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1616, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft message flushes per TiFlash-Proxy instance. Helps identify message processing bottlenecks and uneven Raft communication load distribution across TiFlash nodes, which can indicate network issues or imbalanced cluster workload.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 325, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Coprocessor Response Size", "group": "Follower Read", "used_for": "Used for monitoring the size of data responses returned from TiKV coprocessor requests to TiDB, showing average, median (p50), p80, and p99 percentiles. Essential for diagnosing network bandwidth issues, identifying queries that transfer excessive data, and understanding the I/O characteristics of distributed query execution in follower read scenarios.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_distsql_copr_resp_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.5, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 107, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Rate Limiter Request Wait Duration", "group": "IO Breakdown", "used_for": "Used for monitoring the wait duration of requests throttled by TiKV's rate limiter. Critical for diagnosing performance degradation caused by I/O throttling and identifying when storage bandwidth limits are impacting request latency.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 50, "dashboard": "Dataplane Resource Overview", "title": "Memory Utilization", "group": "Pod", "used_for": "Used for monitoring memory utilization percentage of pods in the dataplane by comparing actual memory usage (working set) against configured resource limits. Helps identify pods approaching memory limits, potential OOM risks, and opportunities for resource allocation optimization.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\"})by(pod)/\nsum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}\n)\n/\nsum by (pod) (\n kube_pod_container_resource_limits{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"POD\"}\n)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 106, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages received by each TiFlash-Proxy instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster coordination issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed in TiFlash-Proxy's raftstore. Critical for diagnosing Raft proposal queuing delays and identifying bottlenecks in TiFlash's replication pipeline that could impact write performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 127, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the time spent appending Raft logs to RocksDB in TiKV's Raft store. Critical for diagnosing write path performance issues and identifying bottlenecks in Raft log persistence that could impact consensus and replication latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 128, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of appending Raft logs to RocksDB on each TiKV instance. Critical for diagnosing Raft replication bottlenecks and identifying slow disk I/O issues that impact consensus performance and data durability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2535, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft log entries are applied in TiFlash-Proxy's raftstore at p95 and p99 percentiles. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "Dataplane Resource Overview", "title": "HPA info(coprocessor-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of coprocessor worker pods, displaying desired, current, and maximum replica counts. Essential for tracking autoscaling behavior and ensuring coprocessor workers scale appropriately to handle query workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572000, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Ingest Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active ingest workers handling global sort operations in TiDB. Helps diagnose data import performance issues and worker resource allocation during large-scale data ingestion tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_ingest_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 4, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "", "used_for": "Used for monitoring TiProxy's memory consumption, including both resident memory and Go heap usage. Essential for detecting memory leaks, identifying out-of-memory risks, and understanding TiProxy's resource utilization patterns during connection proxy operations.", "component": "TIPROXY", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 111, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of applying Raft log entries to TiKV's storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing write path performance issues and identifying when the apply process becomes a bottleneck in the Raft replication pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 110, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft store thread operations in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing Raft log processing bottlenecks and identifying when the store thread becomes a performance bottleneck in the Raft replication pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 119, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of persisting Raft store metadata to disk in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when disk I/O issues are affecting Raft log persistence performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 119, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft read proposals (including local reads and read index operations) per TiFlash instance. Helps identify read request distribution across TiFlash nodes and diagnose potential read hotspots or imbalanced read workloads in the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4194, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiFlash's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate workload imbalances or priority queue bottlenecks affecting read performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 71, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiFlash-Proxy's Raft store. Critical for diagnosing extreme tail latency in region split operations that could impact write performance and cluster stability during data redistribution.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 60, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tikv-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the average CPU utilization of TiKV worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether CPU-based autoscaling is functioning correctly to handle workload changes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\", \n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-default)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 6, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Connection Count", "group": "", "used_for": "Used for monitoring the number of active client connections to TiProxy instances. Essential for tracking connection pool utilization, identifying connection leaks, and diagnosing capacity issues when connection counts approach limits or show unexpected spikes.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 129, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends committing Raft logs to disk. Critical for diagnosing write path performance issues and identifying disk I/O bottlenecks that affect data durability and replication latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2536, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied in TiFlash-Proxy's Raft store, broken down by server instance. Critical for diagnosing apply thread bottlenecks and replication lag issues that can impact TiFlash data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4375, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile performance context duration in TiFlash-Proxy's Raft operations, including both store and apply phases. Critical for diagnosing latency bottlenecks in TiFlash's Raft proposal process and identifying performance degradation in distributed consensus operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "Dataplane Resource Overview", "title": "Scaling Evnt Detials(coprocessor-worker)", "group": "HPA Overview", "used_for": "Used for monitoring Horizontal Pod Autoscaler (HPA) scaling behavior for coprocessor workers, showing current replicas, desired replicas, and the scaling gap. Essential for diagnosing autoscaling responsiveness issues and determining whether coprocessor pods are scaling appropriately to meet workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "last_over_time(kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])", "last_over_time(kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])\n", "(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n -\n kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n) != 0", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval])\n", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset -15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset 15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval])", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset -15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset 15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval])\n", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset -15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset 15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"}[$__interval:$__interval])\n/\nlast_over_time(kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}[$__interval:$__interval])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 126, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Store write send duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of sending Raft store write messages between TiKV instances. Critical for diagnosing network communication delays and identifying slow peers that may impact Raft replication performance and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tidb)", "group": "HPA Overview", "used_for": "Used for monitoring TiDB's Horizontal Pod Autoscaler (HPA) CPU utilization, comparing target thresholds against actual average CPU usage across pods. Essential for validating autoscaling behavior and determining whether TiDB pods are scaling appropriately based on CPU demand.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n", "(\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n /\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n) * 100" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 69, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tikv-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiKV worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for verifying auto-scaling behavior and identifying when TiKV workers are scaling up/down in response to workload changes or hitting configured capacity limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763571999, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Read From Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the read throughput rate from cloud storage during TiDB's global sort operations at different percentiles (p50, p95, p999). Essential for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, helping identify slow reads or inconsistent I/O patterns that could impact large-scale data processing operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_global_sort_merge_sort_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster\", }[2m]))/1024.0/1024.0" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 123, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Process ready duration", "group": "Raft IO", "used_for": "Used for monitoring the time Raft proposals spend waiting in the queue before being processed by TiKV's Raft store thread. Critical for diagnosing Raft processing bottlenecks and identifying when the Raft store thread is overloaded, which can cause write latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Check split", "group": "Raft admin", "used_for": "Used for monitoring the rate of region split checks in TiFlash-Proxy's Raft store, categorized by check type. Helps diagnose region split behavior and identify potential issues with region management or split decision-making in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1975, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft log speed", "group": "Raft propose", "used_for": "Used for monitoring the average rate of Raft log data being proposed in TiFlash-Proxy instances, measured in bytes per second. Helps identify write throughput patterns and potential bottlenecks in Raft log replication for TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 120, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store commit but not persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time gap between when Raft logs are committed and when they are actually persisted to disk in TiKV stores. Critical for diagnosing write durability delays and identifying potential data loss risks during failures, especially when p99/p9999 latencies are high.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 62, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tikv_worker_hint(tikv-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) behavior for TiKV worker pods, comparing the target metric threshold against actual observed values. Essential for understanding autoscaling decisions and diagnosing whether TiKV workers are scaling appropriately based on the custom worker hint metric.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Uptime", "group": "", "used_for": "Used for monitoring the uptime duration of TiProxy instances in seconds. Helps identify recent restarts, instance stability issues, and track how long TiProxy has been running continuously without interruption.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 121, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store commit and persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft log commit and persist operations at the TiKV store level, tracking both p99 and p9999 percentiles. Critical for diagnosing write path bottlenecks and identifying when Raft consensus operations are experiencing delays, which directly impacts transaction commit performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 117, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store before write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the duration of Raft store operations before writing to disk in TiKV, tracking both p99 and p9999 latencies. Critical for diagnosing write path bottlenecks and identifying delays in the Raft log replication process before data persistence.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4198, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiFlash-Proxy's unified read pool at any given time. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient read processing capacity or slow I/O operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763571997, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Write To Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the write throughput rate to cloud storage during TiDB's global sort operations at different percentiles (p50, p95, p999). Essential for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage writes, particularly when troubleshooting slow data import or index building operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_global_sort_merge_sort_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster\", }[2m]))/1024.0/1024.0" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 336, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Active Parallel Upload Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active parallel workers uploading sorted data during TiDB's global sort operations. Essential for diagnosing upload performance bottlenecks and ensuring optimal parallelism during large-scale data import or DDL operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_upload_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 68, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiDB worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for verifying autoscaling behavior and identifying when worker pods are scaling up/down in response to workload changes or hitting configured capacity limits.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4196, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of Level 0 SST files in TiFlash-Proxy's unified read pool RocksDB storage. High Level 0 chance indicates potential read amplification and compaction pressure, helping diagnose I/O performance degradation in TiFlash query processing.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763571996, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Write To Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of writing data to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage, particularly when investigating slow data import or DDL operations using global sort.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 77, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft admin", "used_for": "Used for monitoring the rate of successful Raft admin commands in TiFlash-Proxy, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes that affect TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763571998, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Read From Cloud Storage Duration", "group": "Global Sort", "used_for": "**Used For:**\n\nUsed for monitoring the latency of reading data from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, helping identify slow cloud I/O issues or network latency problems affecting large-scale data processing operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 61, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(coprocessor-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the average CPU utilization of coprocessor-worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for verifying HPA scaling behavior and determining if coprocessor workloads are triggering appropriate auto-scaling actions based on CPU demand.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 1.1", "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 0.9", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-coprocessor)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 63, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tidb_worker_hint(tidb-worker)", "group": "HPA Overview", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) target metrics for TiDB worker pods, comparing the configured spec target against the actual status metric. Helps operators verify that the autoscaling configuration is working correctly and diagnose discrepancies between desired and actual scaling thresholds based on worker hint metrics.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"} *1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "", "used_for": "Used for monitoring TiProxy's CPU utilization rate and maximum available CPU cores. Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiProxy needs vertical scaling or load distribution adjustments.", "component": "TIPROXY", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}[2m])", "tiproxy_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 124, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Process ready duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft log processing readiness in TiKV's Raftstore thread pool, broken down per server instance. Critical for diagnosing Raft replication bottlenecks and identifying instances experiencing delays in processing ready Raft messages, which can impact write performance and consensus operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 130, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft log commit operations on each TiKV instance. Critical for diagnosing write path performance issues and identifying instances with slow disk I/O or Raft replication bottlenecks that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "Dataplane Resource Overview", "title": "HPA Evaluation Result(coprocessor-worker)", "group": "HPA Overview", "used_for": "**Used For:**\n\nUsed for monitoring the Horizontal Pod Autoscaler (HPA) evaluation results for coprocessor worker pods in the dataplane. Helps operators understand autoscaling decisions and verify whether the HPA is correctly scaling coprocessor resources based on workload demands.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 75, "dashboard": "Dataplane Resource Overview", "title": "Scaling Event Distribution(coprocessor-worker)", "group": "HPA Overview", "used_for": "Used for tracking horizontal pod autoscaler (HPA) scaling events for coprocessor workers, showing when replicas scale up or down relative to desired state. Helps diagnose autoscaling behavior and identify whether the coprocessor worker pool is responding appropriately to workload changes.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace" ], "promql": [ "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} > bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])", "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} < bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 76, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft admin", "used_for": "Used for monitoring the rate of Raft administrative operations (configuration changes and leader transfers) in TiFlash-Proxy. Essential for tracking cluster topology changes and diagnosing issues related to region leadership transfers or membership reconfigurations that may impact TiFlash stability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763573001, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Query Network Trasmission Bytes", "group": "Network Transmission", "used_for": "**Used For:**\n\nUsed for monitoring the rate of network data transmission (in bytes per second) during query execution in TiDB, broken down by transmission type. Essential for diagnosing network bottlenecks, identifying queries with excessive data transfer, and understanding the network overhead of distributed query processing across TiDB instances.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_network_transmission{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 73, "dashboard": "Dataplane Resource Overview", "title": "Scale Out summay[WIP](coprocessor-worker)", "group": "HPA Overview", "used_for": "**Used For:**\n\nUsed for monitoring Horizontal Pod Autoscaler (HPA) activity and scale-out events for TiDB coprocessor workers. Helps track when and how frequently the cluster automatically scales coprocessor resources based on CPU utilization, enabling operators to validate autoscaling behavior and capacity planning decisions.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", " kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1", " delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n <=\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) <= 0\n )\n )[$__range:15s]\n))" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 58, "dashboard": "Dataplane Resource Overview", "title": "Avg Mem Util By HPA(tidb)", "group": "HPA Overview", "used_for": "Used for monitoring the average memory utilization of TiDB pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether memory-based autoscaling is triggering appropriately to handle workload demands.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "1", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"} and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 131, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends applying Raft logs to the state machine. Critical for diagnosing write path performance issues, as high apply log duration indicates bottlenecks in persisting committed data to RocksDB, which directly impacts write latency and throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Disconnection OPM", "group": "", "used_for": "Used for monitoring the rate of client disconnections per minute in TiProxy, categorized by disconnection type. Essential for identifying connection stability issues, detecting abnormal disconnect patterns, and diagnosing network or client-side problems affecting database connectivity.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 41, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft proposals are processed in TiFlash-Proxy's raftstore, tracking p95, p99, and average latencies. Critical for diagnosing Raft replication delays and identifying bottlenecks in TiFlash's consensus layer that may cause write stalls or slow data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 118, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the tail latency (p99 and p9999) of the store write end phase in TiKV's Raft replication waterfall. Critical for diagnosing write performance bottlenecks in the final stage of Raft log persistence, helping identify when store-level write operations are experiencing delays that impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the rate of different storage command types executed in TiFlash-Proxy's storage layer. Helps identify command distribution patterns and detect anomalies in specific storage operations like reads, writes, or administrative commands.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 125, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write send duration", "group": "Raft IO", "used_for": "Used for monitoring the time spent sending store write requests in TiKV's Raftstore layer. Critical for diagnosing network or communication bottlenecks between Raft store threads and identifying delays in the write request transmission path.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "Dedicated Dataplane Node Info", "title": "Notice", "group": "", "used_for": "**Used For:**\n\nUsed for displaying important notices, warnings, or configuration information about dedicated dataplane nodes. Provides contextual guidance or alerts that operators should be aware of when monitoring the dataplane infrastructure.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 9, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Goroutine Count", "group": "", "used_for": "Used for monitoring the number of active goroutines in TiProxy instances. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact TiProxy's connection routing performance.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiproxy\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 133, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Client Wait Connection Ready Duration", "group": "Raft IO", "used_for": "Used for monitoring the time Raft clients spend waiting for connections to become ready in TiKV. Critical for diagnosing network connection establishment delays and Raft communication bottlenecks between TiKV nodes that can impact replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14, "dashboard": "Dedicated Dataplane Node Info", "title": "Node labels", "group": "", "used_for": "Used for displaying Kubernetes node labels and metadata for dedicated dataplane nodes. Helps operators verify node configurations, identify node roles, and troubleshoot node-specific issues by correlating system metrics with Kubernetes label information.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "2", "instance", "k8s_cluster", "namespace", "pod" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node, k8s_cluster_info) group_right sum(kube_node_labels) by (node, k8s_cluster_info, label_zone, label_dedicated, label_failure_domain_beta_kubernetes_io_zone, label_beta_kubernetes_io_instance_type)", "(\n label_replace(\n node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"},\n \"node\", \"$0\", \"instance\", \".*\"\n )\n * on(node, k8s_cluster_info) group_right\n sum(kube_node_labels)\n by (node, k8s_cluster_info,\n label_zone, label_dedicated,\n label_failure_domain_beta_kubernetes_io_zone,\n label_beta_kubernetes_io_instance_type)\n)\n* on(node) group_left(pod, component)\n label_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\"$pod\"},\n \"component\", \"$2\", \"pod\",\n \"(^|.*-)(tidb|tikv|tiflash|pd|tso|ticdc)(-[0-9a-z]+)*$\"\n )" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Backend", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency grouped by backend TiDB instances through TiProxy. Essential for identifying which specific backend servers are experiencing slow query performance and diagnosing uneven load distribution or backend-specific issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, backend)), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 46, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets on network interfaces during packet reception on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that cause packet loss and degrade cluster communication performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 134, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Raft Client Wait Connection Ready Duration", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft clients waiting for gRPC connections to become ready between TiKV nodes. Critical for diagnosing network connection establishment issues and inter-node communication delays that can impact Raft replication performance and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "connection", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) )", "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received", "group": "", "used_for": "Used for monitoring the network ingress traffic rate (bytes per second) on dedicated dataplane nodes. Helps identify network bottlenecks, unusual traffic patterns, or potential network saturation issues affecting node performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing slow snapshot generation issues that can impact TiFlash's ability to serve consistent reads and replicate data from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 138, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft Propose", "used_for": "Used for monitoring the distribution of Raft proposals processed per ready batch in TiKV. Helps diagnose Raft processing efficiency and identify whether proposals are being batched effectively, which impacts write throughput and latency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 135, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store io task reschedule", "group": "Raft IO", "used_for": "Used for monitoring the frequency of Raft IO task rescheduling in TiKV's store thread, tracking both region-level reschedules and pending task reschedules. Essential for diagnosing IO thread contention and identifying when the store thread is overloaded, causing tasks to be rescheduled rather than executed immediately.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 167, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler stage operations in TiFlash-Proxy, including both total operations and \"too busy\" rejections per stage. Essential for identifying scheduler bottlenecks and capacity issues when TiFlash-Proxy cannot keep up with incoming requests, helping diagnose write pressure and resource saturation problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 13, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Instance", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency for each TiProxy instance. Essential for identifying performance outliers across TiProxy instances and diagnosing which specific proxy nodes are experiencing elevated query latency issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 20, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Backend Connections", "group": "Balance", "used_for": "Used for monitoring the number of backend connections from TiProxy to TiDB instances. Essential for tracking connection distribution across TiDB nodes and diagnosing connection balancing issues or backend connectivity problems.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_balance_b_conn{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 123, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "0.99 Duration of raft store events", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile latency of Raft store events in TiFlash-Proxy, tracking how long different Raft operations take to complete. Critical for diagnosing replication lag and consensus performance issues in TiFlash's Raft layer, especially when investigating slow data synchronization or write bottlenecks.", "component": "TIFLASH", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 81, "dashboard": "Dedicated Dataplane Node Info", "title": "Component Node Number Trends", "group": "", "used_for": "Used for tracking the number of nodes for each TiDB component (TiDB, TiKV, TiFlash) over time in a dedicated dataplane deployment. Helps monitor cluster scaling activities, detect unexpected node changes, and verify that the desired component topology is maintained.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "3", "namespace" ], "promql": [ "count(\n label_replace(\n kube_pod_info{namespace=\"$namespace\", pod=~\"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"},\n \"component\", \n \"$1$3\", \n \"pod\", \n \"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"\n )\n) by (component)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "CPS by Instance", "group": "Query Summary", "used_for": "Used for monitoring the number of commands per second (CPS) processed by each TiProxy instance. Helps identify load distribution across TiProxy instances and detect performance bottlenecks or imbalanced traffic routing.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 193, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands in TiFlash-Proxy instances. Helps identify scheduling bottlenecks and command queue buildup that could indicate resource contention or processing delays in TiFlash's Raft storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 10, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Owner", "group": "", "used_for": "Used for identifying which TiProxy instance is currently serving as the owner/leader in the cluster. Essential for understanding cluster leadership status and diagnosing issues related to owner election or failover scenarios.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_owner{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 3834, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the volume of data being written by the scheduler in TiFlash-Proxy instances. Helps identify write bottlenecks, uneven write distribution across instances, and potential memory pressure issues in the scheduler's write buffer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 139, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft proposals in TiKV, including both read (local_read) and write (normal) operations. Essential for diagnosing Raft consensus performance, identifying proposal bottlenecks, and understanding the distribution of read/write workload across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 136, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write task block duration per server $optional_quantile", "group": "Raft IO", "used_for": "Used for monitoring the time Raft write tasks spend blocked waiting in the queue on each TiKV server. Critical for diagnosing write bottlenecks and identifying when the Raft store thread is overloaded, which can cause write latency spikes and affect cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiFlash-Proxy. Helps identify priority distribution of I/O operations and diagnose whether high-priority commands are being properly processed or if low-priority commands are causing bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the asynchronous write latency (p95, p99, and average) in TiFlash-Proxy's storage engine. Critical for diagnosing write performance bottlenecks and identifying when TiFlash data ingestion is experiencing delays or degradation.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler commands in TiFlash-Proxy at p95 and p99 percentiles. Essential for diagnosing performance bottlenecks in TiFlash's transaction scheduler operations, helping identify when specific command types experience elevated processing delays that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "Dedicated Dataplane Node Info", "title": "Node Info", "group": "", "used_for": "Used for displaying basic node information in Kubernetes clusters, including node metadata and system details. Helps operators quickly identify node characteristics, OS versions, and cluster membership when troubleshooting infrastructure issues or verifying node configurations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "instance", "k8s_cluster" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node,k8s_cluster_info) group_right kube_node_info" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 18, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Handshake Duration", "group": "Query Summary", "used_for": "Used for monitoring the 95th and 99th percentile latency of client connection handshakes through TiProxy. Critical for diagnosing slow connection establishment issues and identifying authentication or SSL handshake bottlenecks that impact client connection times.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_handshake_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_handshake_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 47, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets during network transmission on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that prevent packets from being sent successfully.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 11, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted", "group": "", "used_for": "Used for monitoring the network transmission rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual outbound traffic patterns, or potential network bottlenecks affecting data replication and client communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 54, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_in_allowance_exceeded", "group": "", "used_for": "Used for monitoring network bandwidth ingress throttling events on dedicated dataplane nodes. Helps identify when incoming network traffic exceeds allocated bandwidth allowances, which can cause packet drops and network performance degradation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_in_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 140, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft read proposals per TiKV server, including local reads and read index operations. Helps diagnose read request distribution across TiKV instances and identify potential read hotspots or imbalanced read workloads in the Raft consensus layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Usage", "group": "Pod", "used_for": "Used for monitoring the working set memory usage of individual pods in the TiDB cluster. Helps identify memory-intensive pods, detect memory leaks, and diagnose out-of-memory (OOM) issues before they impact cluster stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 56, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_out_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring network bandwidth throttling events when outbound traffic exceeds allocated limits on dedicated dataplane nodes. Critical for diagnosing network performance degradation caused by bandwidth quota enforcement and identifying nodes that may need increased network capacity allocation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_out_allowance_exceeded{cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 58, "dashboard": "Dedicated Dataplane Node Info", "title": "PPS_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring packet-per-second (PPS) rate limiting violations on network interfaces. Helps identify when network traffic exceeds configured PPS allowances, which can indicate traffic bursts, potential DDoS attacks, or insufficient network capacity provisioning.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_pps_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 24, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Pending Migration Count", "group": "Balance", "used_for": "Used for monitoring the number of pending connection migrations between TiDB instances managed by TiProxy. Essential for tracking load balancing operations and identifying migration bottlenecks or stuck connections during cluster rebalancing.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "1" ], "promql": [ "label_replace(label_replace(tiproxy_balance_pending_migrate, \"from\", \"$1\", \"from\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\"), \"to\", \"$1\", \"to\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 12, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency through TiProxy at the 95th and 99th percentiles. Essential for identifying performance degradation in client-to-TiDB connections and diagnosing whether slowdowns originate from the proxy layer or backend TiDB servers.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 22, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Duration", "group": "Balance", "used_for": "Used for monitoring the latency of session migration operations in TiProxy during load balancing. Critical for diagnosing performance issues when TiProxy redistributes client connections across TiDB instances, with p95 and p99 percentiles helping identify migration delays that could impact user experience.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_balance_migrate_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_balance_migrate_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 141, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiKV server. Helps identify uneven write distribution across TiKV instances and diagnose potential bottlenecks when specific nodes handle disproportionate Raft proposal loads.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 194, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time transactions spend waiting for latches in TiFlash-Proxy's scheduler, tracking p95, p99, and average wait durations. Critical for diagnosing write contention issues and transaction conflicts that cause scheduling delays in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 282, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Delete Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) delete workers in TiDB. Helps identify performance bottlenecks in the TTL deletion process by showing which phase (scanning, deleting, or waiting) consumes the most time.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"delete_worker\"}[2m])) by (phase)\n" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 132, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of applying Raft logs to the state machine in TiKV instances. Critical for diagnosing write path performance issues and identifying instances experiencing slow log application, which can cause replication lag and affect write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 21, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Session Migration OPM", "group": "Balance", "used_for": "Used for monitoring the rate of session migrations per minute in TiProxy's load balancing operations. Helps diagnose connection rebalancing behavior, identify migration failures, and track which TiDB instances are involved in session redistribution during scaling or failover events.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(label_replace(sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (migrate_res, from, to), \"from\", \"$1\", \"from\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\"), \"to\", \"$1\", \"to\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 168, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiFlash-Proxy, including tracking when the scheduler becomes too busy to process requests. Essential for diagnosing write command bottlenecks and identifying which scheduler stage (e.g., snapshot, apply) is causing performance degradation or request rejections.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "CPS by Backend", "group": "Query Summary", "used_for": "Used for monitoring the number of commands per second (CPS) distributed across different TiDB backend instances through TiProxy. Helps identify load balancing effectiveness and detect if specific backends are receiving disproportionate query traffic or experiencing connection issues.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 142, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft write apply", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft write commands being applied in TiKV's Raftstore. Essential for diagnosing write throughput issues and identifying bottlenecks in the Raft consensus layer's apply process.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "Dedicated Dataplane Node Info", "title": "linklocal_allowance_exceeded", "group": "", "used_for": "Used for monitoring the rate of link-local address allowance exceeded events on network interfaces in the dataplane node. Helps identify network configuration issues or excessive link-local traffic that may indicate connectivity problems or misconfigurations in the Kubernetes cluster networking layer.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_linklocal_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 8, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiFlash-Proxy's storage engine. Critical for identifying storage layer issues, request failures, and potential data access problems that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 17, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring the commands per second (CPS) breakdown by command type in TiProxy. Helps identify which SQL command types (SELECT, INSERT, UPDATE, etc.) are consuming the most traffic and detect unusual command distribution patterns that may indicate application issues or workload changes.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cmd_type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 57, "dashboard": "Dedicated Dataplane Node Info", "title": "conntrack_allowance_exceeded", "group": "", "used_for": "Used for monitoring connection tracking table overflow events on dedicated dataplane nodes. Helps diagnose network connection drops and packet loss caused by exceeding the conntrack table capacity, which is critical for identifying network connectivity issues in high-traffic scenarios.", "component": "", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_conntrack_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 99, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Table Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the connection tracking table utilization ratio on dataplane nodes. Critical for identifying when the conntrack table approaches capacity limits, which can cause connection drops and network issues in high-traffic Kubernetes environments.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "node_nf_conntrack_entries{cluster=\"$k8s_cluster\",instance=~\"$instance\"} / node_nf_conntrack_entries_limit{cluster=\"$k8s_cluster\",instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 143, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft change_set apply", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft change set applications in TiKV's storage engine. Helps diagnose Raft log application performance and identify bottlenecks in the state machine replication process when investigating write throughput issues or replication lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kvengine_apply_change_set_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 373, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose write amplification issues and identify commands that are writing unexpectedly large numbers of keys, which could impact TiFlash storage performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Reasons", "group": "Balance", "used_for": "Used for tracking the reasons why TiProxy migrates client sessions between TiDB instances during load balancing operations. Helps diagnose balance behavior and identify whether migrations are triggered by backend failures, maintenance operations, or load rebalancing policies.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 146, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft proposals spend waiting in queue before being processed in TiKV. Critical for diagnosing write latency issues caused by proposal queue congestion or raftstore thread saturation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 144, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft change set proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft configuration change proposals (membership changes like adding/removing peers) per TiKV instance. Essential for tracking cluster topology changes and diagnosing issues related to region peer adjustments or replica rebalancing operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 195, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose I/O efficiency issues by identifying commands that scan excessive keys, which may indicate inefficient queries or potential performance bottlenecks in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 560, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiFlash-Proxy's scheduler, broken down by operation tags. Essential for diagnosing scan performance issues, identifying inefficient scan patterns, and understanding the workload characteristics of specific scheduler commands in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 145, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft change set apply per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft change set apply operations per TiKV server instance. Helps identify imbalanced Raft proposal workloads across TiKV nodes and diagnose potential bottlenecks in distributed consensus operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set|shard_ingest\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 675, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations on the lock column family within TiFlash-Proxy's scheduler for specific command types. Helps diagnose lock-related scanning performance issues and identify potential bottlenecks in transaction conflict detection or lock cleanup operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 26, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Get Backend Duration", "group": "Backend", "used_for": "Used for monitoring the latency of TiProxy selecting and connecting to TiDB backend instances, showing p95 and p99 percentiles. Critical for diagnosing connection establishment delays and backend selection performance issues that could impact client query response times.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_backend_get_backend_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_backend_get_backend_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 1615, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages sent by each TiFlash-Proxy instance to other nodes in the cluster. Helps identify communication bottlenecks, network issues, or instances with abnormally high/low message traffic that could indicate replication problems or cluster imbalance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 54, "dashboard": "Dataplane Resource Overview", "title": "Root Disk Usage", "group": "Node", "used_for": "**Used For:**\n\nUsed for monitoring root filesystem disk space utilization across cluster nodes. Essential for preventing disk full conditions that could cause node failures, service disruptions, or data write failures.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "1 - sum(node_filesystem_avail_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device) / sum(node_filesystem_size_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 147, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Propose wait duration per server", "group": "Raft Propose", "used_for": "**Used For:**\n\nUsed for monitoring the 99th and 99.99th percentile wait time before Raft proposals are processed on each TiKV server. Critical for diagnosing Raft layer bottlenecks and identifying instances experiencing proposal queue congestion, which can cause write latency spikes and affect transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Ping Backend Duration", "group": "Backend", "used_for": "Used for monitoring the latency of health check pings from TiProxy to backend TiDB servers. Essential for detecting network issues or unresponsive TiDB instances that could affect connection routing and failover decisions.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_backend_ping_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 28, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Health Check Cycle", "group": "Backend", "used_for": "Used for monitoring the health check cycle duration of TiProxy backend connections to TiDB instances. Essential for diagnosing connection pool health and detecting delays in backend availability detection that could impact connection routing and failover responsiveness.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_backend_health_check_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 100, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Insert Failures", "group": "", "used_for": "Used for monitoring the rate of connection tracking table insertion failures on dataplane nodes. Critical for diagnosing network connection issues caused by conntrack table exhaustion, which can lead to dropped connections and increased latency when the system cannot track new network flows.", "component": "", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_insert_failed{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 300, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Table Count By TTL Schedule Delay", "group": "TTL", "used_for": "Used for monitoring the number of tables grouped by their TTL (Time To Live) scheduling delay watermark in TiDB. Helps identify tables experiencing delays in TTL job scheduling, which is critical for diagnosing data cleanup performance issues and ensuring timely expiration of time-sensitive data.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_server_ttl_watermark_delay{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", type=\"schedule\"}) by (type, name)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 829, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the detailed scan operations on TiFlash-Proxy's write column family during scheduler command execution. Helps diagnose write-related performance issues by tracking scan patterns and identifying potential bottlenecks in key-value scan operations within the write CF.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 830, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the distribution of key-value scan operations in TiFlash-Proxy's default column family, broken down by operation tags. Helps diagnose scan performance issues and understand the characteristics of read workloads processed by the scheduler.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 31, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of data received by TiProxy from backend TiDB servers, measured in bytes per second. Helps identify backend traffic patterns, detect network bottlenecks, and diagnose imbalanced data distribution across TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 150, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft log entries spend waiting in the apply queue before being applied to the state machine in TiKV. Critical for diagnosing write path bottlenecks and identifying when the apply worker threads are overloaded or falling behind.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "Dedicated Dataplane Node Info", "title": "Usage Per Core", "group": "CPU", "used_for": "Used for monitoring CPU utilization per individual core on dedicated dataplane nodes. Helps identify CPU core imbalances, hotspots, and whether workloads are properly distributed across all available cores.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "sum by (cpu) (irate(node_cpu_seconds_total{ job=\"node-exporter\", mode!=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m]))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 29, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Dial Backend Fails OPM", "group": "Backend", "used_for": "Used for monitoring the rate of failed backend connection attempts from TiProxy to TiDB servers, measured in operations per minute (OPM). Critical for identifying connectivity issues, backend availability problems, or network failures between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_backend_dial_backend_fail{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 45, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk I/O Util", "group": "Disk", "used_for": "Used for monitoring disk I/O utilization percentage on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting saturated storage devices, and diagnosing performance degradation caused by excessive I/O operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 148, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft store write tasks spend waiting in the queue before being processed in TiKV. Critical for diagnosing write bottlenecks and identifying when the Raft store thread is overloaded, which can cause increased write latency and affect overall cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent by TiFlash-Proxy nodes. Essential for tracking data replication activity and diagnosing issues with region snapshot transfers, which can indicate slow learner additions or region recovery operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1395, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy worker queues, broken down by worker name. Essential for identifying task queue bottlenecks and worker thread saturation that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 152, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write handle msg duration", "group": "Raft Propose", "used_for": "Used for monitoring the time TiKV's store thread spends handling write-related messages in the Raft consensus process. Critical for diagnosing Raft proposal bottlenecks and identifying when the store thread becomes a performance bottleneck for write operations.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1876, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiFlash-Proxy's future pool, broken down by pool name. Helps identify task processing throughput and potential bottlenecks in TiFlash's asynchronous task execution system.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 59, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiFlash-Proxy. Helps identify worker thread activity patterns and diagnose task processing bottlenecks or imbalances across worker pools.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 38, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots in TiFlash-Proxy by type (sending/receiving/applying). Essential for diagnosing data replication issues, identifying snapshot transfer bottlenecks, and understanding cluster data synchronization patterns during region migrations or node recoveries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 35, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Cross Location Bytes/Second", "group": "Traffic", "used_for": "Used for monitoring the rate of cross-location network traffic in bytes per second handled by TiProxy instances. Essential for identifying cross-region data transfer costs and diagnosing network bottlenecks in geo-distributed TiDB deployments.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_traffic_cross_location_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 34, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets sent from TiProxy instances to backend TiDB servers. Helps identify network traffic distribution patterns, detect connection issues, and diagnose potential packet loss or network bottlenecks between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 7, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "**Used For:**\n\nUsed for monitoring the current memory utilization on dedicated dataplane nodes. Essential for identifying memory pressure, potential OOM risks, and capacity planning for node resources.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(\n (\n (\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n )\n / node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n ) * 100)\n" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 1877, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy's future pool, broken down by pool name. Helps identify task queue buildup and potential thread pool saturation issues that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 33, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the outbound network traffic rate from TiProxy to backend TiDB servers, measured in bytes per second. Helps identify network bandwidth usage patterns, detect traffic imbalances across backends, and diagnose connectivity or throughput issues between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 6, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "Used for monitoring memory consumption on dedicated dataplane nodes, breaking down total memory usage into active, buffer, and cache components. Essential for identifying memory pressure, potential OOM risks, and understanding how the node allocates memory between application workloads and system caches.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "node", "port" ], "promql": [ "max(\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n)\n", "max(node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$node:$port\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 98, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Drops", "group": "", "used_for": "Used for monitoring the rate of dropped connection tracking entries in the Linux kernel's netfilter conntrack subsystem. Critical for diagnosing network connectivity issues caused by conntrack table exhaustion, which can lead to dropped connections and failed network requests when the system reaches its connection tracking limits.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 154, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft propose speed", "group": "Raft Propose", "used_for": "Used for monitoring the rate at which Raft proposals are being generated in TiKV, measured by the total size of Raft logs proposed per second. Essential for diagnosing write throughput issues and identifying bottlenecks in the Raft consensus layer when write performance degrades.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average time spent waiting for disk read and write operations to complete on dedicated dataplane nodes. Essential for identifying disk I/O bottlenecks and diagnosing storage performance degradation that could impact TiDB cluster operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 4, "dashboard": "Dedicated Dataplane Node Info", "title": "CPU Utilization", "group": "CPU", "used_for": "Used for monitoring the overall CPU utilization percentage on dedicated dataplane nodes. Essential for identifying CPU bottlenecks, resource saturation, and determining if nodes need scaling or workload rebalancing.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m])) * 100)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 44, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile size of Raft snapshots in TiFlash-Proxy. Critical for identifying abnormally large snapshot transfers that could impact network bandwidth and replication performance during region recovery or scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 151, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Apply wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th and 99.99th percentile wait time before Raft log entries are applied to the state machine in TiKV. Critical for diagnosing write path bottlenecks when the apply worker thread pool is overloaded or experiencing scheduling delays, which directly impacts write latency and transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 159, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "rfstore batch message count", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft message batch sizes received by TiKV stores. Helps diagnose Raft communication efficiency and identify potential network batching issues that could impact replication performance.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n rfstore_receive_msgs_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk iops", "group": "Disk", "used_for": "Used for monitoring disk I/O operations per second (IOPS) on dedicated dataplane nodes, tracking both read and write operations. Essential for identifying disk bottlenecks, understanding storage workload patterns, and diagnosing performance degradation caused by excessive disk activity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 36, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiFlash-Proxy, including sending, applying, and generating snapshots. Critical for diagnosing slow replica synchronization issues and identifying performance bottlenecks in TiFlash's Raft snapshot handling process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 32, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets received by TiProxy from backend TiDB servers. Helps identify network throughput patterns, backend communication issues, and potential packet loss or congestion between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 160, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "rfstore batch message size", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft message batch sizes received by TiKV stores. Helps diagnose network efficiency and identify issues with Raft message batching, which can impact replication performance and network bandwidth utilization.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n rfstore_receive_msgs_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring disk I/O bandwidth (read and write rates) on dedicated dataplane nodes. Helps identify disk bottlenecks, unusual I/O patterns, and storage performance issues that may impact TiDB cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 54, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Available size", "group": "", "used_for": "Used for monitoring the available storage capacity on each TiFlash instance. Critical for preventing storage exhaustion and planning capacity expansion before TiFlash nodes run out of disk space.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 43, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pair counts in TiFlash-Proxy snapshots. Helps identify extreme cases where snapshots contain unusually large numbers of KV pairs, which can indicate data skew issues or potential performance bottlenecks during snapshot operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 149, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Store write wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th and 99.99th percentile wait duration for Raft store write tasks on each TiKV instance. Critical for diagnosing write bottlenecks in the Raft store layer, particularly when investigating tail latency issues or identifying specific TiKV nodes experiencing delayed write task processing before data is persisted to RocksDB.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 153, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store write trigger size", "group": "Raft Propose", "used_for": "Used for monitoring the size distribution of write batches that trigger RocksDB writes in TiKV's Raft store. Helps diagnose write amplification issues and optimize batch write thresholds when investigating storage I/O performance problems.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 155, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft Propose", "used_for": "Used for monitoring the performance context duration in TiKV's Raft store and apply operations. Helps diagnose low-level storage engine performance issues by tracking time spent in RocksDB operations during Raft proposal processing, essential for identifying bottlenecks in write path latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 157, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft Process", "used_for": "Used for monitoring the rate of Raft ready messages being handled in TiKV's Raft consensus process. Essential for diagnosing Raft I/O throughput and identifying bottlenecks in consensus message processing that could impact write performance and replication efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average time spent waiting for disk write operations to complete on dataplane nodes. Essential for identifying disk I/O bottlenecks and diagnosing storage performance degradation that could impact TiKV write throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 53, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Store size", "group": "", "used_for": "Used for monitoring the storage space consumed by each TiFlash instance, including both local and remote storage usage. Essential for tracking storage capacity trends, identifying instances approaching disk limits, and planning storage expansion before running out of space.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"\"}) by (instance)", "sum(tiflash_system_current_metric_StoreSizeUsedRemote{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 91, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk iops", "group": "Disk", "used_for": "Used for monitoring write I/O operations per second on disk devices across dedicated dataplane nodes. Helps identify disk write bottlenecks, unusual I/O patterns, and storage performance degradation that could impact TiDB write operations.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2, "dashboard": "Dedicated Dataplane Node Info", "title": "System load", "group": "CPU", "used_for": "Used for monitoring the 1-minute, 5-minute, and 15-minute system load averages on dedicated dataplane nodes. Essential for identifying CPU resource saturation and determining if the node is overloaded or experiencing sustained high workload pressure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(node_load1{job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load5{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load15{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 21, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Uptime", "group": "", "used_for": "Used for monitoring the continuous running time of TiFlash instances since their last restart. Helps quickly identify recent restarts, instance stability issues, and verify cluster uptime during troubleshooting or after maintenance operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2108, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of threads in TiFlash-Proxy instances, showing both per-state breakdown and total thread counts. Essential for diagnosing thread pool saturation, identifying blocked or waiting threads, and detecting I/O bottlenecks that may impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2258, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes per second) across different TiFlash-Proxy threads, filtered to show only threads with I/O rates exceeding 1KB/s. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or imbalanced workload distribution in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 200, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region", "group": "", "used_for": "Used for monitoring the distribution of Raft regions and hibernated peer states across TiFlash instances. Essential for diagnosing data replication imbalances, region scheduling issues, and identifying instances with abnormal region counts that may impact query performance or storage capacity.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_proxy_tikv_raftstore_hibernated_peer_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}) by (instance, state)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 158, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Max duration of raft store events", "group": "Raft Process", "used_for": "Used for monitoring the maximum (p999999) latency of critical Raft store events in TiKV, including event processing and broadcast operations. Essential for diagnosing extreme tail latency issues in Raft consensus operations that could indicate store thread contention, slow disk I/O, or network problems affecting cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 999999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 161, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "rfstore main worker time breakdown", "group": "Raft Process", "used_for": "Used for monitoring the time distribution of different operations in TiKV's Raftstore main worker thread, including message processing, message receiving, and auxiliary worker synchronization. Critical for diagnosing Raft consensus performance bottlenecks and identifying which specific operation is causing delays in the Raft replication pipeline.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_store_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_receive_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_sync_aux_worker_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_send_aux_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_process_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Replica read lock checking duration", "group": "Raft Process", "used_for": "Used for monitoring the time spent checking locks during replica read operations in TiKV. Helps diagnose performance issues with follower reads and identify lock contention that may be blocking read requests from replica nodes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Memory", "group": "", "used_for": "Used for monitoring TiFlash memory usage across the cluster, including proxy process resident memory, total memory capacity, and jemalloc retained memory. Essential for identifying memory pressure, potential OOM risks, and optimizing TiFlash resource allocation.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}", "sum(tiflash_system_current_metric_MemoryCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$instance\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 55, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Capacity size", "group": "", "used_for": "Used for monitoring the total storage capacity allocated to each TiFlash instance. Essential for capacity planning and identifying instances approaching storage limits before they impact query performance or data ingestion.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 51, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "", "used_for": "Used for monitoring CPU utilization of TiFlash instances, including both the TiFlash proxy process and total logical CPU cores available. Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiFlash nodes need scaling when handling analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m])", "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2661, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches in TiFlash-Proxy threads, which occur when threads are preempted by the OS scheduler. High rates (>50/sec) indicate CPU contention or resource starvation, helping diagnose performance degradation caused by excessive thread competition or insufficient CPU resources.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 50" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 93, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring the write bandwidth (bytes per second) to physical disks on dedicated dataplane nodes. Essential for identifying disk I/O bottlenecks, detecting write-heavy workloads, and diagnosing storage performance issues that may impact TiKV or other data-intensive components.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2660, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring voluntary context switches in TiFlash-Proxy threads, which occur when threads willingly yield CPU (typically during I/O waits). High rates (>200/sec) indicate threads are frequently waiting for I/O operations, helping diagnose I/O bottlenecks and thread scheduling inefficiencies in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 200" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 7, "dashboard": "[SharedScope]TiProxy-Summary-Cluster-Clinic", "title": "Create Connection OPM", "group": "", "used_for": "Used for monitoring the rate of new connection creation per minute on TiProxy instances. Helps identify connection churn patterns, detect connection storms, and diagnose issues with client connection behavior or connection pooling inefficiencies.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_create_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 163, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Peer msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft message sizes exchanged between TiKV peers. Helps diagnose network bandwidth issues, identify abnormally large Raft messages, and troubleshoot replication performance problems caused by message size anomalies.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of individual read and write operations on disk devices in the dataplane node. Helps identify I/O patterns and detect inefficient small I/O operations that could indicate fragmentation or suboptimal workload characteristics.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 164, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the distribution of Raft apply message batch sizes in TiKV. Helps diagnose Raft log application efficiency and identify whether messages are being batched optimally, which impacts replication performance and I/O patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 138, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the efficiency of read operations in TiFlash-Proxy's RocksDB storage engine, tracking hits across different cache layers (memtable, block cache) and storage levels (L0). Essential for diagnosing read performance issues and understanding cache effectiveness in TiFlash-Proxy's storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 181, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "IO Throughput", "group": "", "used_for": "Used for monitoring the I/O throughput (bytes per second) of TiFlash proxy threads across instances. Essential for identifying I/O bottlenecks, diagnosing storage performance issues, and understanding data read/write patterns in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 92, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of disk I/O operations (both read and write) per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and potential bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 165, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Enable apply unpersisted log region count", "group": "Raft Process", "used_for": "**Used For:**\n\nUsed for monitoring the number of regions that have enabled the unpersisted apply log feature in TiKV. Helps track Raft log optimization adoption across regions and diagnose potential data consistency or recovery issues when this performance feature is active.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 129, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring RocksDB seek operation rates in TiFlash-Proxy, including total seeks, successful seeks, and next operations. Essential for diagnosing read performance issues and understanding data access patterns in TiFlash's storage engine, particularly when investigating slow queries or high I/O overhead.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 48, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the percentage of used storage across different devices. Essential for preventing disk full scenarios and identifying nodes approaching storage capacity limits that could impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "((max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) /\n max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) * 100" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 49, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Used", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the difference between total filesystem size and available space. Essential for identifying storage capacity issues and preventing disk full conditions that could impact node operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "(max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 30, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read (rrqm) and write (wrqm) requests per second on disk devices in the dataplane node. Helps assess I/O efficiency and identify whether the kernel is successfully merging adjacent I/O requests to reduce disk operations and improve performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 167, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Poll Handle Duration", "group": "Raft Process", "used_for": "Used for monitoring the 99th percentile duration of Raft apply operations in TiKV's Raftstore. Critical for diagnosing slow log application issues that can cause replication lag and affect write performance across the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_running_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 141, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "SST Import Service", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST import service threads, which handle importing SST files during data replication and recovery. Essential for diagnosing performance bottlenecks during bulk data ingestion, region migration, or when TiFlash replicas are being synchronized from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"sst_importer.*\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 95, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read and write requests per second on disk devices in dedicated dataplane nodes. Helps identify disk I/O efficiency and whether the kernel is successfully combining adjacent I/O requests, with higher merge rates indicating better I/O optimization and reduced disk overhead.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 145, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region Task", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash region task threads, which handle region-related operations like Raft log application and region metadata management. Essential for diagnosing TiFlash performance bottlenecks related to region synchronization and identifying when region tasks are consuming excessive CPU resources.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 125, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiFlash-Proxy, showing maximum, P99, and P95 durations. Critical for diagnosing read performance issues when TiFlash queries need to locate specific keys in the storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 154, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "SST Apply", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST (Sorted String Table) apply threads, which handle applying Raft log entries to the storage engine. Critical for diagnosing write performance bottlenecks and identifying when TiFlash instances are struggling to keep up with data replication from TiKV.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 166, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply ahead of persistence raft log count", "group": "Raft Process", "used_for": "Used for monitoring the number of Raft log entries that have been applied to the state machine but not yet persisted to disk in TiKV. Critical for diagnosing data durability risks and understanding the gap between in-memory state and durable storage, especially during recovery scenarios or when investigating potential data loss issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiFlash-Proxy's RocksDB storage engine, tracking successful writes (self/other), write timeouts, and WAL writes. Essential for diagnosing write throughput issues, identifying write timeout problems, and understanding write-ahead logging activity in TiFlash storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 8, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Helps identify read-heavy workloads, diagnose storage performance bottlenecks, and verify if EBS volume IOPS limits are being reached during high read activity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 62, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance", "used_for": "Used for monitoring when AWS EBS volumes exceed their provisioned IOPS or throughput limits on dedicated dataplane nodes. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to I/O throttling and degraded database performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 137, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations in TiFlash-Proxy's RocksDB storage engine. Essential for diagnosing write durability overhead and identifying potential I/O bottlenecks when WAL sync frequency becomes excessive or irregular.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 151, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Small Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background threads that handle small maintenance tasks like compaction and cleanup. Essential for diagnosing whether background operations are consuming excessive CPU resources and impacting foreground query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 82, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation latency in TiFlash-Proxy, showing maximum, P99, and P95 get durations in microseconds. Essential for diagnosing TiFlash read performance issues and identifying slow storage layer operations that may impact analytical query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 174, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Vote", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft vote messages sent between TiKV nodes during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 155, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Store", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of Raft Store threads in TiFlash instances, which handle Raft consensus protocol operations for data replication. Essential for diagnosing performance bottlenecks in TiFlash's data synchronization layer and identifying instances with excessive Raft processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 52, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption on TiKV and TiFlash persistent volume claims in Kubernetes environments. Essential for tracking storage utilization and preventing disk full issues that could cause data write failures or service disruptions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 147, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash region worker threads, which handle region data synchronization from TiKV. Essential for diagnosing performance bottlenecks in TiFlash's data replication process and identifying instances with excessive region management overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 128, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiFlash-Proxy's storage engine. Essential for diagnosing storage I/O performance issues, identifying compaction bottlenecks, and understanding write amplification patterns that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 130, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) operations in TiFlash-Proxy's RocksDB storage engine, tracking maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues that could impact TiFlash data synchronization and durability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 126, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write operation latency in TiFlash-Proxy's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash write performance bottlenecks and identifying when data replication from TiKV experiences delays or slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 135, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the Write-Ahead Log (WAL) fsync latency in TiFlash-Proxy's RocksDB storage engine, tracking maximum, P99, and P95 sync durations. Critical for diagnosing write performance bottlenecks and disk I/O issues that impact TiFlash data durability and write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 149, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Manual Compaction", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's manual compaction thread pool. Essential for diagnosing performance issues related to manual compaction operations and identifying whether compaction threads are becoming a bottleneck during data reorganization tasks.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 96, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption of TiDB Persistent Volume Claims (PVCs) in Kubernetes environments. Essential for tracking storage utilization and preventing disk space exhaustion issues that could impact TiDB server operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 156, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Large Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background threads handling large tasks like compaction and data reorganization. Essential for diagnosing performance bottlenecks when TiFlash experiences high background processing load or storage layer slowdowns.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 51, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiKV and TiFlash persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized deployments.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 136, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiFlash-Proxy, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage performance issues, as prolonged compaction times can indicate write amplification problems or resource bottlenecks affecting query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 175, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages that are dropped or failed to send between TiKV nodes. Critical for diagnosing cluster communication issues, network problems, or overloaded nodes that prevent proper Raft replication and consensus.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 171, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft message flushes per TiKV server instance. Helps diagnose Raft communication efficiency and identify instances with abnormal message batching or network flush patterns that could impact cluster consensus performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 170, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages sent from each TiKV server to other nodes in the cluster. Essential for diagnosing Raft communication issues, network bottlenecks, and identifying servers with abnormally high or low message transmission rates that could indicate replication problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 161, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Client", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of gRPC asynchronous client threads in TiFlash instances. Helps diagnose communication bottlenecks and resource contention when TiFlash interacts with other TiDB components through gRPC connections.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\", instance=~\"$tiflash_role\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 172, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages received by each TiKV server. Helps identify communication imbalances, network issues, or servers experiencing abnormal Raft message traffic that could indicate replication problems or cluster instability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 97, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiDB persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized TiDB deployments.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 140, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring SST (Sorted String Table) file read latency in TiFlash-Proxy's RocksDB storage engine, tracking maximum, P99, and P95 read durations. Essential for diagnosing TiFlash storage performance issues and identifying slow disk I/O or SST file access bottlenecks that impact analytical query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 178, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Message Receive Delay duration", "group": "Raft Message", "used_for": "Used for monitoring the time delay between when Raft messages are received and when they are processed in TiKV instances. Critical for diagnosing Raft consensus performance issues, network bottlenecks, or processing backlogs that could impact cluster replication and data consistency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 168, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Poll Handle Batch Count", "group": "Raft Process", "used_for": "Used for monitoring the 99th percentile batch size of Raft apply messages handled by TiKV's apply worker threads. Critical for diagnosing Raft apply efficiency and identifying whether apply operations are being properly batched, which directly impacts write throughput and replication performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 64, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring write input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Essential for identifying storage bottlenecks, evaluating whether workloads are approaching EBS IOPS limits, and diagnosing write-heavy performance issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 66, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum write I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume write operations are experiencing delays that could impact TiKV write performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 173, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft messages sent between TiKV nodes, categorized by message status (accept/drop). Essential for diagnosing Raft communication issues, network problems, or cluster consensus delays that could impact data replication and availability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 179, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Raft Message Receive Delay Duration", "group": "Raft Message", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft message reception in TiKV instances. Critical for diagnosing consensus layer communication delays and identifying network or processing bottlenecks that could impact cluster replication performance and data consistency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum read I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume read operations are experiencing high latency that could impact TiKV read performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 102, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB block cache in TiFlash-Proxy instances, broken down by column family. Essential for diagnosing memory pressure issues and optimizing cache configuration when TiFlash-Proxy experiences performance degradation or OOM problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 267, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Segment Scheduler", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash segment scheduler threads, which manage data segment operations like merging, splitting, and compaction. Essential for diagnosing performance bottlenecks in TiFlash's background maintenance tasks and identifying instances where segment scheduling is consuming excessive CPU resources.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 177, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Raft Message Send Wait Duration", "group": "Raft Message", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of Raft message sending in TiKV, measuring the wait time before messages are dispatched to other nodes. Critical for diagnosing Raft consensus delays, network bottlenecks, or thread pool saturation that could impact replication performance and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 467, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write traffic flow through TiFlash-Proxy's RocksDB block cache, including cache insertion rates. Essential for diagnosing cache efficiency issues, identifying excessive cache churn, and optimizing memory usage in TiFlash storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 256, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FAP builder", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's FAP (Fast Add Peer) builder threads, which handle data replication and region snapshot building. Essential for diagnosing performance bottlenecks during data synchronization, replica addition, or when TiFlash nodes are catching up with TiKV data changes.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"fap_builder.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\", instance=~\"$tiflash_role\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 153, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Sender", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash snapshot sender threads, which handle sending Raft snapshots to other replicas during data replication. Critical for diagnosing performance issues during region rebalancing, replica recovery, or when adding new TiFlash nodes to the cluster.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 257, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Apply Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's apply worker threads, which handle Raft log application from TiKV. Essential for diagnosing write performance bottlenecks and identifying when apply threads are saturated, which can cause replication lag between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\", name!~\"apply_low_.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\", name!~\"apply_low_.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 159, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Server", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async server threads (async pollers) that handle asynchronous network operations. Essential for diagnosing whether gRPC communication bottlenecks or high CPU consumption in async processing threads are impacting TiFlash's ability to handle client requests efficiently.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\", instance=~\"$tiflash_role\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 65, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the write throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify write bandwidth bottlenecks, verify if workloads are approaching EBS volume throughput limits, and diagnose I/O performance issues in cloud storage.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 88, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the memtable hit rate in TiFlash-Proxy's RocksDB engine, showing the percentage of read requests served directly from memory without accessing disk. Essential for evaluating memory cache effectiveness and diagnosing read performance issues when the hit rate drops unexpectedly.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 295, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Local Index Pool", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's local index thread pool, which handles local index operations. Essential for diagnosing performance bottlenecks in index processing and identifying whether the local index thread pool is saturated or underutilized during query execution.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\", instance=~\"$tiflash_role\"})", "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexSched*\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 75, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB Cluster Pods", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for displaying all TiDB cluster pods running in the specified Kubernetes namespace. Helps operators quickly identify pod distribution, verify deployment status, and correlate pod-level issues with EBS volume performance metrics in dedicated dataplane nodes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "namespace" ], "promql": [ "kube_pod_info{namespace=\"$namespace\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 181, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft administrative operations (configuration changes, leader transfers, etc.) in TiKV. Essential for diagnosing cluster topology changes, member additions/removals, and leader rebalancing activities that may impact cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 132, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiFlash-Proxy's RocksDB storage engine, including keys read, keys written, and corrupt keys detected during compaction. Essential for diagnosing data flow issues, identifying write/read imbalances, and detecting potential data corruption problems in TiFlash storage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2451, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons triggering RocksDB compactions in TiFlash-Proxy's storage engine, broken down by column family and compaction trigger type. Essential for diagnosing storage I/O patterns and understanding whether compactions are triggered by write volume, file count limits, or other factors affecting TiFlash-Proxy performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 268, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Segment Reader", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SegmentReader threads, which handle reading data segments from storage. Critical for diagnosing read performance bottlenecks and identifying whether segment reading operations are CPU-bound during query execution.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\", instance=~\"$proxy_instance\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\", instance=~\"$proxy_instance\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 90, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits at the cluster level. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to I/O throttling and degraded database performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 184, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft Admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiKV's Raftstore. Critical for diagnosing extreme tail latency in region split operations, which can indicate issues with region management, load imbalance, or storage bottlenecks affecting cluster scalability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 176, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Message Send Wait duration", "group": "Raft Message", "used_for": "Used for monitoring the time Raft messages spend waiting in the send queue before being transmitted between TiKV nodes. Critical for diagnosing network congestion, message backlog issues, or bottlenecks in Raft consensus communication that can impact cluster coordination and replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 468, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiFlash-Proxy's RocksDB storage engine, including overall block additions, data block additions, and filter block additions. Essential for understanding cache population patterns and diagnosing issues related to cache efficiency and memory pressure in TiFlash storage nodes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 76, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait time for disk write operations on TiKV nodes' EBS volumes. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk writes at the infrastructure level.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 80, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the block cache hit ratio in TiFlash-Proxy's RocksDB storage engine, including overall block cache, data blocks, and filter blocks. Essential for evaluating cache effectiveness and diagnosing read performance issues—low hit rates indicate insufficient cache size or poor cache utilization causing excessive disk I/O.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 182, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft admin commands being applied in TiKV, excluding compaction operations. Essential for diagnosing cluster configuration changes, region splits/merges, and other administrative operations that may impact cluster stability or performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 183, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Check split", "group": "Raft Admin", "used_for": "Used for monitoring the rate of region split checks in TiKV's Raftstore. Helps diagnose region splitting activity and identify potential issues with automatic region management when regions grow beyond size thresholds.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 187, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Peer in Flashback State", "group": "Raft Admin", "used_for": "Used for monitoring the number of Raft peers currently in flashback state across TiKV instances. Essential for tracking the progress and scope of flashback operations, helping diagnose issues when restoring the cluster to a previous point in time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the average disk read wait latency on TiKV nodes at the EBS volume level. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk reads in cloud-deployed TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 79, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk wait latency", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait latency per I/O operation for EBS volumes in dedicated dataplane nodes, tracking both read and write operations separately. Essential for identifying storage performance bottlenecks and diagnosing slow disk I/O issues that could impact cluster performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)", "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 333, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring the rate of voluntary context switches across TiFlash proxy threads. Helps identify threads waiting for I/O operations or resources, which can indicate I/O bottlenecks or resource contention affecting TiFlash performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, name) > 200" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 331, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes read/written per second) across TiFlash proxy threads. Essential for identifying I/O bottlenecks, understanding storage access patterns, and diagnosing performance issues related to disk operations in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m])) by (name, io, $additional_groupby) > 1024" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 329, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of TiFlash proxy threads across instances, showing both per-state breakdown and total thread counts. Essential for diagnosing thread pool saturation, identifying instances with excessive blocked or waiting threads, and detecting I/O bottlenecks in TiFlash's proxy layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}) by (instance, state)", "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 85, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of TiFlash-Proxy's RocksDB storage engine, tracking both direct bytes read and iterator-based reads. Essential for diagnosing TiFlash read performance issues, identifying I/O bottlenecks, and understanding data access patterns during analytical query execution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 78, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average disk write wait latency for TiFlash nodes at the cluster level. Essential for identifying storage I/O bottlenecks and diagnosing slow write performance issues in TiFlash's columnar storage engine, particularly when investigating query performance degradation or data ingestion delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(node)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 86, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring write I/O throughput in TiFlash-Proxy's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written. Essential for diagnosing write performance bottlenecks, identifying excessive write amplification, and understanding storage I/O patterns in TiFlash's proxy layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 186, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Load base split duration", "group": "Raft Admin", "used_for": "Used for monitoring the latency of load-based region split operations in TiKV. Critical for diagnosing performance issues when TiKV automatically splits regions due to high load, helping identify bottlenecks in the split process that could impact write throughput and cluster scalability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 90, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction I/O throughput in TiFlash-Proxy, tracking bytes read during compaction, bytes written after compaction, and flush write operations. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification in TiFlash's storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 191, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log GC write operations", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection write operations in TiKV. Helps identify GC activity levels and potential bottlenecks when cleaning up old Raft logs, which is critical for managing disk space and maintaining cluster health.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 85, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP In Use", "group": "", "used_for": "Used for monitoring the number of TCP sockets currently in use on dedicated dataplane nodes. Helps identify socket exhaustion issues, connection leaks, or abnormal connection patterns that could impact network communication and service availability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "node_sockstat_TCP_inuse{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 335, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring the rate of involuntary context switches in TiFlash threads, which occur when the OS forcibly preempts threads due to CPU contention or time slice expiration. High values indicate CPU saturation or resource competition, helping diagnose performance degradation caused by insufficient CPU resources or thread scheduling issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, name) > 50" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Executor QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second handled by different TiFlash coprocessor executors (e.g., table scan, aggregation, join). Essential for understanding TiFlash query workload distribution and identifying which executor types are processing the most requests, helping diagnose performance bottlenecks in analytical queries.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Request Duration", "group": "Coprocessor", "used_for": "Used for monitoring coprocessor request latency in TiFlash at multiple percentiles (p80, p95, p99, p999). Essential for identifying slow analytical queries and diagnosing TiFlash performance degradation, especially when OLAP workloads experience increased response times.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 134, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiFlash-Proxy's RocksDB engine, showing maximum, P99, and P95 values. Helps diagnose write amplification issues and identify whether write operations are batching efficiently or performing many small writes that could impact I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 131, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys in each column family of TiFlash-Proxy's RocksDB storage engine. Helps assess data distribution across column families and identify potential storage imbalances or unexpected data growth patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 13, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Request Handle Duration", "group": "Coprocessor", "used_for": "Used for monitoring the latency distribution of coprocessor request handling in TiFlash at various percentiles (p80, p95, p99, p999). Essential for identifying slow analytical query processing and diagnosing TiFlash performance bottlenecks when OLAP workloads experience degraded response times.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 193, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log lag", "group": "Raft Log", "used_for": "Used for monitoring the lag between Raft log replication and application in TiKV instances. Critical for identifying replication delays that can indicate network issues, slow followers, or overloaded nodes affecting data consistency and cluster health.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 190, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log GC kv sync duration", "group": "Raft Log", "used_for": "Used for monitoring the 99.99th percentile latency of syncing Raft logs to TiKV's key-value storage engine. Critical for diagnosing extreme tail latency in Raft log persistence, which can indicate disk I/O bottlenecks or storage performance degradation affecting consensus replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 89, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP Connections", "group": "", "used_for": "Used for monitoring the current number of established TCP connections on dedicated dataplane nodes. Helps identify connection leaks, abnormal connection spikes, or resource exhaustion issues that could impact network communication and service availability.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance" ], "promql": [ "node_netstat_Tcp_CurrEstab{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 87, "dashboard": "Dedicated Dataplane Node Info", "title": "Segments retransmitted", "group": "", "used_for": "Used for monitoring the rate of TCP segment retransmissions on dedicated dataplane nodes. Helps diagnose network reliability issues, packet loss, or congestion problems that may impact cluster communication and overall performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "rate(node_netstat_Tcp_RetransSegs{instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 516, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiFlash-Proxy instances. Helps identify potential memory pressure or snapshot accumulation issues that could impact read performance and resource utilization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 189, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log GC write duration", "group": "Raft Log", "used_for": "Used for monitoring the 99.99th percentile latency of Raft log garbage collection write operations in TiKV. Critical for diagnosing tail latency issues in Raft log cleanup that could impact write performance and storage management, especially when log compaction becomes a bottleneck.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 518, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiFlash-Proxy's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O during read operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 127, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of pending compaction bytes in TiFlash-Proxy's RocksDB storage engine, broken down by column family. Essential for identifying compaction backlogs and I/O pressure that could lead to write stalls or performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 195, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log GC failed", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection failures in TiKV instances. Critical for identifying issues with log cleanup that could lead to disk space exhaustion and degraded cluster performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 185, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Load base split event", "group": "Raft Admin", "used_for": "Used for monitoring the frequency and types of load-based region split events in TiKV. Helps diagnose whether regions are being split appropriately based on load patterns, which is critical for identifying hotspot issues and evaluating automatic load balancing effectiveness.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Request QPS", "group": "Coprocessor", "used_for": "Used for monitoring the rate of coprocessor requests per second handled by TiFlash, broken down by request type. Essential for tracking TiFlash query workload patterns and identifying sudden changes in analytical query traffic or specific request type bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22303, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Resolved Ts Lag", "group": "Summary", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with real-time changes, helping diagnose replication bottlenecks or downstream processing issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Error QPS", "group": "Coprocessor", "used_for": "Used for monitoring the rate of coprocessor request errors in TiFlash, broken down by error reason. Essential for identifying and diagnosing issues with query execution failures, helping pinpoint whether errors stem from timeout, region unavailability, or other coprocessor-related problems.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (reason)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 77, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average read latency of EBS volumes serving TiFlash nodes in dedicated dataplane deployments. Critical for identifying storage I/O bottlenecks that impact TiFlash's analytical query performance, especially when read operations experience delays at the disk level.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 22302, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Checkpoint Lag", "group": "Summary", "used_for": "Used for monitoring the replication lag between the source TiDB cluster and TiCDC's checkpoint position, measured in time units. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with upstream changes in real-time replication scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 196, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log fetch", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log entry fetch operations in TiKV. Helps diagnose replication lag and identify followers that are frequently fetching logs from leaders, which may indicate slow followers or network issues affecting Raft consensus.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n rfstore_raft_entry_fetches_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the data volume per read operation in TiFlash-Proxy's RocksDB storage engine, showing maximum, P99, and P95 metrics. Helps diagnose read amplification issues and identify inefficient read patterns that may indicate suboptimal data layout or excessive small reads.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 14, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Response Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the rate of response data bytes sent from TiFlash coprocessor to TiDB per second, broken down by request type. Helps identify data transfer bottlenecks and diagnose network bandwidth issues between TiFlash and TiDB during query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 192, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log GC seek operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of seek operations performed during Raft log garbage collection in TiKV. Helps diagnose performance issues related to log cleanup efficiency and identify potential bottlenecks in the Raft log GC process that could impact storage reclamation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 194, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log gc skipped", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection operations that were skipped in TiKV. Helps identify issues where log compaction is being delayed or prevented, which can lead to excessive disk usage and potential performance degradation in the Raft consensus layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 18999, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Upstream Write Bytes / s", "group": "Summary", "used_for": "Used for monitoring the rate of bytes written per second from TiCDC to upstream TiDB/TiKV. Helps identify data throughput bottlenecks and diagnose replication performance issues when CDC is writing change data back to the source cluster.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",scope=~\"general\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 63, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Cop task memory usage", "group": "Coprocessor", "used_for": "Used for monitoring memory consumption of coprocessor tasks in TiFlash at various percentiles (p80, p95, p99, p999). Essential for identifying memory-intensive analytical queries and diagnosing potential out-of-memory issues or memory pressure in TiFlash's coprocessor execution layer.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 863, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression efficiency of different RocksDB levels in TiFlash-Proxy's storage engine. Helps identify storage optimization opportunities and diagnose issues where poor compression ratios lead to excessive disk space usage or increased I/O overhead.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 165, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Exchange Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the rate of data exchange between TiFlash nodes during distributed query execution, measured in bytes per second by exchange type. Essential for diagnosing network bottlenecks and data shuffle performance issues in MPP (Massively Parallel Processing) queries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_exchange_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 100, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the number of active RPC threads in TiFlash instances handling coprocessor requests. Essential for identifying thread pool saturation and diagnosing I/O bottlenecks when TiFlash struggles to process incoming requests from TiDB.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2381, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency of RocksDB stall condition changes in TiFlash-Proxy's storage engine across different column families. Critical for diagnosing write throttling issues and identifying when the storage engine enters or exits stall states due to memtable limits, L0 file accumulation, or pending compaction bytes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 73, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Bandwidth", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring disk read and write bandwidth across EBS volumes in the dedicated dataplane cluster. Essential for identifying disk I/O bottlenecks, detecting abnormal storage traffic patterns, and ensuring EBS volumes are not approaching their throughput limits during heavy workloads.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])", "rate(node_disk_written_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 20198, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiCDC Input Bytes / s", "group": "Summary", "used_for": "Used for monitoring the rate of incoming data bytes per second being written to TiCDC's event store, broken down by instance. Essential for tracking data ingestion throughput and identifying instances with abnormal input rates or potential bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_write_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2002, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in TiFlash-Proxy's RocksDB storage engine for each column family. Essential for diagnosing compaction issues, storage amplification problems, and identifying when LSM-tree structure becomes unbalanced due to write pressure or compaction lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 101, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Early Drops", "group": "", "used_for": "Used for monitoring the rate of connection tracking table entries dropped due to table exhaustion on dataplane nodes. Critical for diagnosing network connection limits and identifying when the conntrack table is full, which can cause new connection failures and packet drops.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_early_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 517, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiFlash-Proxy's RocksDB engine. Critical for identifying long-running transactions or snapshots that may block compaction, cause storage bloat, and degrade read/write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Handling Request Number", "group": "Coprocessor", "used_for": "Used for monitoring the number of coprocessor requests currently being handled by TiFlash instances, categorized by request type. Essential for identifying request processing bottlenecks, understanding workload distribution, and diagnosing performance issues when TiFlash appears overloaded or unresponsive.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 197, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft log async fetch task duration", "group": "Raft Log", "used_for": "Used for monitoring the 99.99th percentile latency of asynchronous Raft log fetch tasks in TiKV, along with average duration and pending task count. Critical for diagnosing extreme tail latency in Raft log replication and identifying bottlenecks in the async-read-worker that could impact consensus performance and data synchronization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"async-read-worker\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 199, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Resource Usage", "group": "Raft Engine", "used_for": "Used for monitoring memory and disk resource consumption by Raft Engine in TiKV instances. Helps identify resource usage patterns and potential memory leaks or excessive disk space usage in the Raft log storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n raft_engine_resouce_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20003, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Event Row Count / s", "group": "Summary", "used_for": "Used for monitoring the rate of DML (Data Manipulation Language) events being written to downstream sinks per second in TiCDC. Essential for tracking replication throughput and identifying bottlenecks in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_dml_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2452, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons why RocksDB write operations are being stalled in TiFlash-Proxy. Essential for diagnosing write performance bottlenecks by identifying whether stalls are caused by memtable limits, L0 file accumulation, pending compaction bytes, or other RocksDB flow control mechanisms.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 102, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads", "group": "Coprocessor", "used_for": "Used for monitoring the current thread count across different thread pools in TiFlash instances, excluding RPC and maximum limit threads. Essential for identifying thread resource exhaustion, thread pool saturation, or abnormal thread creation that could impact TiFlash coprocessor performance and I/O operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 200, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Entry Count", "group": "Raft Engine", "used_for": "Used for monitoring the average number of entries stored in TiKV's Raft Engine, categorized by entry type. Helps track Raft log accumulation and identify potential issues with log compaction or replication lag that could impact storage efficiency and recovery time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_total_entries_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22301, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "The Status of Changefeeds", "group": "Summary", "used_for": "Used for monitoring the operational status of TiCDC changefeeds across keyspaces. Essential for quickly identifying which changefeeds are running, stopped, or experiencing issues, enabling rapid troubleshooting of data replication problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 103, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Max Threads", "group": "Coprocessor", "used_for": "Used for monitoring the maximum thread count limits configured in TiFlash instances across different thread pools. Helps identify thread pool capacity constraints and potential resource bottlenecks when actual thread usage approaches these maximum thresholds.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 67, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify I/O bottlenecks, diagnose storage performance issues, and verify whether read-intensive workloads are approaching EBS volume throughput limits.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 10049, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Dispatcher Count", "group": "Summary", "used_for": "Used for monitoring the total number of active table event dispatchers across TiCDC instances in the new architecture. Helps identify dispatcher distribution imbalances and potential bottlenecks when tracking replication workload across changefeeds and tables.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanager_table_event_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (instance, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2003, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST file ingestion operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing snapshot application performance issues and identifying bottlenecks when TiFlash replicas receive data from TiKV during replication or recovery.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 157, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "MPP Query count", "group": "Coprocessor", "used_for": "Used for monitoring the count of MPP (Massively Parallel Processing) queries being executed on TiFlash instances. Helps identify query distribution across TiFlash nodes and detect potential workload imbalances or query processing bottlenecks in analytical workloads.", "component": "TIFLASH", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 202, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read/Write Duration $optional_quantile", "group": "Raft Engine", "used_for": "Used for monitoring the latency of Raft Engine's read and write operations at a configurable quantile. Critical for diagnosing performance bottlenecks in TiKV's Raft log storage, including write persistence delays and entry fetch latencies that can impact consensus and replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_fetch_entries_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4453, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiFlash-Proxy's key management system. Helps verify encryption key storage health and diagnose issues with data-at-rest encryption functionality in TiFlash.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 12025, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Maintainer Checkpoint Lag", "group": "Lag Summary", "used_for": "Used for monitoring the replication lag between TiCDC maintainer's checkpoint timestamp and the current time. Critical for identifying data synchronization delays and ensuring downstream systems receive changes within acceptable latency thresholds.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_maintainer_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 101, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Max Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the maximum number of RPC threads configured in TiFlash instances. Helps identify thread pool capacity limits and diagnose whether RPC thread exhaustion is causing request queuing or processing delays.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 203, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flow", "group": "Raft Engine", "used_for": "Used for monitoring the write throughput and batch sizes in TiKV's Raft Engine. Helps diagnose write performance issues by tracking the rate of data being written to Raft logs, which is critical for understanding replication overhead and identifying potential bottlenecks in the consensus layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 201, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Operation", "group": "Raft Engine", "used_for": "Used for monitoring the operation rate of Raft Engine's core I/O activities, including log persistence and entry fetching operations per second. Essential for diagnosing Raft Engine throughput issues and identifying bottlenecks in TiKV's consensus log storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n raft_engine_fetch_entries_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 103, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables in TiFlash-Proxy, broken down by column family. Essential for diagnosing memory pressure issues and understanding write buffer usage that can impact write performance and memory stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 199, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Time of the Longest Live MPP Task", "group": "Coprocessor", "used_for": "Used for monitoring the execution time of the longest currently running MPP (Massively Parallel Processing) task in TiFlash. Helps identify stuck or abnormally long-running analytical queries that may indicate performance bottlenecks or resource contention in distributed query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_mpp_task_monitor{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4497, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the size of encryption metadata files in TiFlash-Proxy. Helps track storage overhead from encryption key management and identify potential issues with metadata file growth that could impact disk space or I/O performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 719, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Memory Quota", "group": "Summary", "used_for": "Used for monitoring memory quota usage in TiCDC's dynamic stream processing across different modules (event collector, event service, table trigger dispatcher). Essential for identifying memory pressure issues and potential OOM risks in TiCDC changefeeds, helping diagnose performance degradation or replication lag caused by memory constraints.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", area=~\"$changefeed\", instance=~\"$ticdc_instance\", module=~\"event-collector\"}) by (keyspace_name, area, instance, module, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12029, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "EventService Resolved Ts Lag ", "group": "Lag Summary", "used_for": "Used for monitoring the resolved timestamp lag in TiCDC's EventService, tracking both received and sent events. Critical for identifying replication delays and diagnosing data synchronization issues between upstream TiDB and downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_service_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=~\"received\"}) by (instance, type)", "sum(ticdc_event_service_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=~\"sent\"}) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22471, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "EventStore Resolved Ts Lag ", "group": "Lag Summary", "used_for": "Used for monitoring the resolved timestamp lag in TiCDC's EventStore component, which indicates how far behind the EventStore is from the current time in processing change data. Critical for identifying replication delays and ensuring timely data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_store_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 208, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Other Durations $optional_quantile", "group": "Raft Engine", "used_for": "Used for monitoring the latency of miscellaneous Raft Engine operations including snapshot creation and backup processes at a configurable percentile. Helps diagnose performance bottlenecks in Raft Engine's auxiliary operations that may impact overall TiKV stability and recovery capabilities.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_take_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_backup_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12026, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "DispatcherManager Checkpoint Lag", "group": "Lag Summary", "used_for": "Used for monitoring the checkpoint timestamp lag in TiCDC's DispatcherManager, which indicates how far behind the replication is from the source database. Critical for identifying replication delays and ensuring data synchronization meets recovery point objectives (RPO) in change data capture scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "ticdc_dispatchermanager_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 205, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Bytes / Written", "group": "Raft Engine", "used_for": "Used for monitoring the distribution of write batch sizes in TiKV's Raft Engine, showing p99, p9999, and average bytes written per batch. Essential for diagnosing write amplification issues and understanding Raft log write patterns that impact storage efficiency and write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 100, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Storage capacity", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity available across the entire TiDB cluster. Essential for capacity planning and identifying when the cluster is approaching storage limits that could impact write operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 207, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "File Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of Write-Ahead Log (WAL) files in TiKV's Raft Engine, including both total WAL count and files pending compaction. Essential for diagnosing Raft Engine storage issues, detecting compaction lag, and identifying potential disk space problems caused by WAL file accumulation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_pending_compaction_wals\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "avg((\n raft_engine_total_wals_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 299, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Establish calldata details", "group": "Coprocessor", "used_for": "Used for monitoring the count of different types of established call data objects in TiFlash's coprocessor communication layer. Helps diagnose connection pooling efficiency and identify potential memory leaks or resource accumulation issues in the gRPC communication between TiDB/TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_establish_calldata_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type != \"new_request_calldata\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 218, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Puller Output Events / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events pulled from TiKV by TiCDC's puller component, broken down by instance and event type. Essential for tracking data ingestion throughput and identifying bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 111, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Queries Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting queries in TiFlash's task scheduler across different resource groups. Essential for identifying query queuing issues, resource contention, and determining if TiFlash is overloaded or experiencing scheduling bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"waiting_queries_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"active_queries_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4496, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether TiFlash-Proxy's encryption-at-rest feature has been successfully initialized. Essential for confirming encryption status during cluster startup and troubleshooting encryption configuration issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 12022, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Maintainer Resolved Ts lag", "group": "Lag Summary", "used_for": "Used for monitoring the replication lag between TiCDC's maintainer resolved timestamp and the current time. Critical for identifying data synchronization delays in change data capture pipelines and ensuring near real-time replication to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_maintainer_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 109, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Estimated Thread Usage and Limit", "group": "Task Scheduler", "used_for": "Used for monitoring TiFlash's task scheduler thread consumption against configured soft and hard limits. Critical for diagnosing thread exhaustion issues, identifying when TiFlash is approaching resource constraints, and preventing query failures due to insufficient thread resources.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"thread_soft_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"thread_hard_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"global_estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"group_entry_count\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12027, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "DispatcherManager Resolved Ts Lag", "group": "Lag Summary", "used_for": "Used for monitoring the replication lag between TiCDC's DispatcherManager resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring near real-time change data capture performance across changefeeds.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "ticdc_dispatchermanager_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 102, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current storage used", "group": "", "used_for": "Used for monitoring the current storage utilization percentage across the TiDB cluster. Helps identify when the cluster is approaching capacity limits and needs scaling or data cleanup to prevent storage exhaustion issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 204, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write Duration Breakdown $optional_quantile", "group": "Raft Engine", "used_for": "Used for monitoring the breakdown of Raft Engine write operation latency at a specified percentile, including WAL write time, write throttle duration, and apply duration. Essential for diagnosing write performance bottlenecks in TiKV's Raft Engine and identifying whether delays occur during log writing, throttling, or applying operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4498, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the latency of reading and writing encryption metadata files in TiFlash-Proxy, showing p95 and maximum durations. Critical for diagnosing performance issues related to encryption-at-rest operations, particularly when encrypted TiFlash instances experience slow I/O or metadata access delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 113, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Tasks Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting tasks in TiFlash's task scheduler across different resource groups. Essential for identifying task queue bottlenecks, resource contention, and scheduling delays that may impact query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"waiting_tasks_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"active_tasks_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 209, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Log GC Size", "group": "Raft Engine", "used_for": "Used for monitoring the size of Raft logs being garbage collected in TiKV's Raft Engine, broken down by GC status. Helps diagnose Raft log accumulation issues and verify that log cleanup is functioning properly to prevent disk space exhaustion.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_log_file_gc_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (status, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 103, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Normal stores", "group": "", "used_for": "Used for monitoring the total number of healthy TiKV stores currently online in the PD cluster. Essential for verifying cluster capacity and quickly detecting store failures or unavailability issues that could impact data availability and performance.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_up_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 166, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Data size in send and receive queue", "group": "Coprocessor", "used_for": "Used for monitoring the amount of data queued in TiFlash's exchange operator send and receive buffers during distributed query execution. Helps identify data transfer bottlenecks and memory pressure in MPP query processing when queue sizes grow excessively.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_exchange_queueing_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 211, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Worker Status", "group": "Raft Engine", "used_for": "Used for monitoring the health status of Raft Engine's DFS (Disk Flush Service) worker threads in TiKV. Critical for detecting worker thread failures that could impact Raft log persistence and data durability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n raft_engine_dfs_worker_healthy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 206, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Duration of WAL Operation - $optional_quantile", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency of Raft Engine's Write-Ahead Log (WAL) operations, including truncate, rotate, and compact operations in TiKV. Critical for diagnosing Raft log management performance issues and identifying bottlenecks in consensus protocol operations that could impact write throughput and replication stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_truncate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_compact_wal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 101, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current storage size", "group": "", "used_for": "Used for monitoring the total storage capacity currently used across the entire TiDB cluster as reported by PD. Essential for tracking storage growth trends and planning capacity expansion before running out of disk space.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12035, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "EventCollector Resolved Ts Lag", "group": "Lag Summary", "used_for": "Used for monitoring the lag distribution of resolved timestamps in TiCDC's EventCollector component. Critical for identifying replication delays and diagnosing data synchronization issues between upstream TiDB and downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dispatcher_received_event_lag_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 107, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Min TSO", "group": "Task Scheduler", "used_for": "Used for monitoring the minimum timestamp oracle (TSO) value being processed by TiFlash's task scheduler across different resource groups. Helps identify data freshness issues and diagnose delays in TiFlash receiving or processing data from TiKV, which could indicate replication lag or scheduling bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"min_tso\"}) by (instance, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 220, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "EventService Output Event Row / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's EventService component, measured in rows per second. Helps identify event processing throughput, detect bottlenecks in the data replication pipeline, and diagnose issues when change data capture performance degrades.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_send_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 297, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Network Transmission", "group": "Coprocessor", "used_for": "Used for monitoring the network transmission rate (bytes per second) in TiFlash instances, broken down by transmission type. Essential for identifying network bottlenecks, diagnosing data transfer issues between TiFlash and other components, and detecting abnormal network traffic patterns that may impact coprocessor query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_network_transmission_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiFlash-Proxy's RocksDB engine, tracking maximum, P99, and P95 latencies when writes are temporarily blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when TiFlash storage layer experiences backpressure that could impact data ingestion from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 210, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of backup operations performed by Raft Engine in TiKV, categorized by status. Helps track backup activity patterns and identify potential issues with Raft Engine's log backup mechanism, which is critical for data durability and recovery.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n raft_engine_backup_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (status, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 213, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Request Rate", "group": "Raft Engine", "used_for": "Used for monitoring the rate of DFS (Distributed File System) requests in TiKV's Raft Engine. Helps identify I/O throughput patterns and diagnose potential bottlenecks in Raft log storage operations.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_dfs_requests_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20000, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Write Bytes / s", "group": "Summary", "used_for": "Used for monitoring the rate of data being written to downstream sinks (in bytes per second) by TiCDC changefeeds. Essential for tracking replication throughput, identifying bandwidth bottlenecks, and diagnosing slow data synchronization issues between upstream TiDB and downstream systems.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_write_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance, type, changefeed)", "avg_over_time(sum(rate(ticdc_sink_write_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance, type, changefeed)[1m:])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 19, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Schema Internal DDL OPM", "group": "DDL", "used_for": "Used for monitoring the operations per minute (OPM) of internal DDL operations in TiFlash, broken down by DDL type and instance. Essential for tracking schema synchronization activity from TiDB to TiFlash and diagnosing issues with table replication or DDL processing delays.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (type)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m]))", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (type,instance)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 229, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Puller Output Event Rows", "group": "Dataflow", "used_for": "**Used For:**\n\nUsed for monitoring the number of change data capture events pulled from TiKV by TiCDC's puller component. Essential for tracking data replication throughput and diagnosing CDC pipeline bottlenecks or stalls in the event pulling stage.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "ticdc_kvclient_pull_event_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 115, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Waiting Duration", "group": "Task Scheduler", "used_for": "Used for monitoring the time tasks spend waiting in TiFlash's task scheduler queue before execution, tracking p80, p90, and p100 percentiles. Critical for diagnosing TiFlash query performance issues caused by resource contention or scheduler bottlenecks when tasks experience excessive queuing delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(1.00, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,le,resource_group))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 106, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Abnormal stores", "group": "", "used_for": "Used for monitoring the health status of TiKV stores in the cluster by tracking disconnected, unhealthy, and low-space stores. Critical for quickly identifying storage node failures, network issues, or capacity problems that could impact cluster availability and data reliability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_disconnected_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_unhealth_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_low_space_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_down_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_offline_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_tombstone_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_slow_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 18, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply OPM", "group": "DDL", "used_for": "Used for monitoring the rate of schema change operations applied to TiFlash per minute, categorized by operation type. Essential for tracking DDL activity impact on TiFlash nodes and diagnosing schema synchronization issues or DDL-related performance bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 117, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Hard Limit Exceeded Count", "group": "Task Scheduler", "used_for": "Used for monitoring how often TiFlash's task scheduler hits hard resource limits and rejects tasks. Critical for identifying resource exhaustion issues and determining when TiFlash nodes need capacity scaling or configuration adjustments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"hard_limit_exceeded_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 214, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Upload Rate", "group": "Raft Engine", "used_for": "Used for monitoring the rate of data uploaded to distributed file system (DFS) by Raft Engine in TiKV instances. Helps diagnose backup performance issues and track data transfer efficiency when TiKV persists Raft logs to remote storage.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 104, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Number of Regions", "group": "", "used_for": "Used for monitoring the total number of leader regions across the PD cluster. Essential for tracking cluster data distribution, identifying region imbalances, and diagnosing scheduling issues that may impact I/O performance and cluster stability.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"leader_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 212, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Uploaded Bytes to DFS", "group": "Raft Engine", "used_for": "Used for monitoring the total bytes uploaded from Raft Engine to the Distributed File System (DFS). Helps track data replication volume and diagnose issues with remote storage synchronization or backup operations in TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current peer count", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total number of Regions currently managed by the PD cluster. Essential for tracking cluster data distribution scale and identifying abnormal Region count changes that may indicate split/merge issues or data migration problems.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20199, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "EventService Output Event Rows", "group": "Dataflow", "used_for": "Used for monitoring the number of event rows sent by TiCDC's EventService component, broken down by instance and event type. Essential for tracking data replication throughput and identifying bottlenecks or anomalies in the event distribution pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_service_send_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}) by (instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4492, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiFlash-Proxy's storage layer. Helps verify encryption-at-rest is working correctly and track the volume of encrypted data files across the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 12000, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Received Event Rows / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of event rows received by TiCDC's event collector per second, broken down by instance and event type. Essential for tracking data change capture throughput and identifying potential bottlenecks or anomalies in the CDC replication pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dispatcher_received_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10064, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Received Event Rows", "group": "Dataflow", "used_for": "Used for monitoring the number of event rows received by TiCDC's event collector, broken down by instance and event type. Helps diagnose data replication throughput issues and identify bottlenecks in the change data capture pipeline when events aren't being processed as expected.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatcher_received_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}) by (instance,type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 107, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region health", "group": "", "used_for": "Used for monitoring the health status and distribution of regions across the TiDB cluster. Essential for identifying region-related issues such as missing peers, pending peers, down peers, or learner regions that may impact data availability and cluster stability.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 310, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "CPU Usage (irate)", "group": "Imbalance read/write", "used_for": "Used for monitoring CPU utilization rate across TiFlash instances to identify imbalanced workload distribution. Helps diagnose whether certain TiFlash nodes are overloaded while others remain underutilized, indicating potential data skew or uneven query routing issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "irate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$tiflash_role\"}[2m])", "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 108, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Leader/Primary", "group": "", "used_for": "Used for identifying the current leader/primary member in the PD (Placement Driver) cluster. Essential for verifying cluster leadership status and diagnosing leader election issues or split-brain scenarios during troubleshooting.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "service_member_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 219, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Seek operations", "group": "Kv Engine", "used_for": "Used for monitoring the rate of seek operations performed by TiKV's storage engine (RocksDB). Essential for diagnosing read performance issues and understanding storage layer access patterns, as high seek rates may indicate inefficient queries or suboptimal data layout.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_seek_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 665, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Flush Rows", "group": "Dataflow", "used_for": "Used for monitoring the total number of rows flushed to downstream sinks by TiCDC changefeeds. Essential for tracking data replication throughput and identifying changefeeds with abnormal flush patterns or replication bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 309, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Segment Reader", "group": "Imbalance read/write", "used_for": "Used for monitoring CPU usage and thread count of TiFlash segment reader threads across instances. Helps identify imbalanced read workload distribution and potential I/O bottlenecks when certain TiFlash nodes show disproportionately high segment reader activity compared to others.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 217, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Get operations", "group": "Kv Engine", "used_for": "Used for monitoring the rate of get operations performed on TiKV's key-value storage engine. Helps identify read operation frequency and diagnose potential read hotspots or abnormal access patterns in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_get_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 109, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Offline store progress", "group": "", "used_for": "Used for monitoring the progress of store removal operations in PD. Helps track when stores are being taken offline and diagnose issues with slow or stalled decommissioning processes.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"removing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 19, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring the uptime duration of TiCDC, TiKV, and PD components in the cluster. Essential for tracking service availability, identifying recent restarts or crashes, and verifying cluster stability during troubleshooting.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*tikv.*\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*pd.*\"})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 218, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Get duration", "group": "Kv Engine", "used_for": "Used for monitoring the 99.99th percentile latency of KV engine get operations in TiKV. Critical for identifying extreme tail latency in point lookups and diagnosing storage layer read performance issues that affect query responsiveness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "avg((\n kv_engine_get_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 220, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "Kv Engine", "used_for": "Used for monitoring the 99.99th percentile latency of RocksDB seek operations in TiKV's storage engine. Critical for diagnosing extreme tail latency in point lookups and range scans, helping identify storage layer performance bottlenecks when queries experience occasional severe slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "avg((\n kv_engine_seek_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 275, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiCDC processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiCDC service failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 222, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "Kv Engine", "used_for": "Used for monitoring the 99.99th percentile write latency in TiKV's storage engine, tracking extreme tail latency for write operations. Critical for diagnosing write performance degradation and identifying storage bottlenecks that affect the slowest write requests in the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "avg((\n kv_engine_write_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Throughput by instance", "group": "Imbalance read/write", "used_for": "Used for monitoring read throughput across TiFlash instances, including file descriptor reads, PSM reads, and background reads. Essential for identifying imbalanced read workloads and detecting instances with disproportionately high or low read traffic that may indicate data skew or performance bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 316, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Request QPS by instance", "group": "Imbalance read/write", "used_for": "Used for monitoring the rate of coprocessor requests per second handled by each TiFlash instance, broken down by request type. Essential for identifying workload imbalances across TiFlash nodes and diagnosing uneven read/write distribution that could lead to hotspots or resource bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 24, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiCDC server processes and comparing it against the configured CPU limit (GOMAXPROCS). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiCDC needs additional CPU resources to handle replication workload efficiently.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}[2m])", "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 225, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Keys flow", "group": "Kv Engine", "used_for": "Used for monitoring the rate of key operations flowing through TiKV's storage engine. Helps identify write-heavy workloads, detect unusual key operation patterns, and diagnose performance issues related to excessive key modifications or deletions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiCDC's memory consumption, tracking both resident memory (RSS) and Go heap allocation. Essential for detecting memory leaks, identifying memory pressure issues, and determining if TiCDC nodes need resource scaling or optimization.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 223, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write flow", "group": "Kv Engine", "used_for": "Used for monitoring the write throughput (bytes per second) to TiKV's storage engine across cluster instances. Essential for identifying write-heavy workloads, diagnosing I/O bottlenecks, and understanding data ingestion patterns that may impact storage performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS By Instance", "group": "Imbalance read/write", "used_for": "Used for monitoring write command operations per second across TiFlash instances, broken down by command type. Helps identify write workload imbalances between instances and diagnose uneven data distribution or hotspot issues in the TiFlash cluster.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, type)", "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 112, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current scaling speed", "group": "", "used_for": "Used for monitoring the current rate at which PD is scaling the cluster, measured in operations per time unit. Helps operators track the speed of data rebalancing and region scheduling activities, essential for understanding cluster adjustment progress during scale-out/scale-in operations or load rebalancing.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 26, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of goroutines and OS threads in TiCDC instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate memory issues or performance degradation in the change data capture process.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ " go_goroutines{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}", "go_threads{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", component=~\".*ticdc.*\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 238, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Mark Cache and Minmax Index Cache Memory Usage", "group": "Memory trace", "used_for": "Used for monitoring memory consumption of TiFlash's mark cache, minmax index cache, and MVCC index cache. Essential for diagnosing TiFlash memory usage patterns and identifying potential memory pressure issues related to data indexing and caching structures.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_asynchronous_metric_RNMVCCIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 325, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Number of Keyspaces", "group": "Memory trace", "used_for": "Used for monitoring the number of keyspaces currently managed by TiFlash instances. Helps track multi-tenancy resource allocation and identify potential memory overhead from keyspace metadata, especially useful when diagnosing memory usage patterns in multi-tenant TiDB deployments.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_NumKeyspace{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 324, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Number of Physical Tables", "group": "Memory trace", "used_for": "Used for monitoring the number of physical tables in TiFlash, including delta-merge storage tables and total storage instances. Helps track memory footprint related to table metadata and diagnose issues when excessive table counts lead to memory pressure or performance degradation.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_DT_NumStorageDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_NumIStorage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 227, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Block Cache Miss", "group": "Kv Engine", "used_for": "Used for monitoring the rate of block cache misses in TiKV's storage engine. Helps diagnose memory pressure and storage performance issues when cache hit rates are low, indicating potential need for cache size tuning or identifying hot data access patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 312, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Throughput By Instance", "group": "Imbalance read/write", "used_for": "Used for monitoring write and ingest throughput distribution across TiFlash instances to identify imbalanced workloads. Essential for detecting hotspot instances with disproportionately high write traffic that may cause performance bottlenecks or resource exhaustion.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write\"}[2m])) by (instance)", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"ingest\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 202, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store limit", "group": "Cluster", "used_for": "Used for monitoring the configured rate limits on TiKV stores, which control the maximum number of operations (like adding/removing peers) allowed per store. Essential for diagnosing scheduling bottlenecks and understanding why PD may be throttling region operations during cluster rebalancing or recovery.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [], "promql": [ "pd_cluster_store_limit" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 111, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Left time", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the estimated time remaining for PD cluster operations such as region scheduling, leader balancing, or other rebalancing tasks. Helps operators understand when ongoing cluster adjustments will complete and assess scheduling progress during maintenance or recovery scenarios.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 323, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Bytes of MemTables", "group": "Memory trace", "used_for": "Used for monitoring memory consumption of TiFlash's DeltaTree MemTables, tracking both actual usage and allocated memory. Essential for diagnosing TiFlash memory pressure issues and understanding write buffer memory overhead during data ingestion.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_DT_BytesMemTable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_BytesMemTableAllocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 204, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region label isolation level", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of regions across different label isolation levels in PD's placement rules. Helps verify that regions are properly distributed according to configured label constraints (e.g., rack, zone, host) and diagnose placement policy violations or imbalanced region distribution across failure domains.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 224, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "Kv Engine", "used_for": "Used for monitoring the volume of data written per write operation in TiKV's storage engine, tracking both average and tail percentiles (p99, p9999). Essential for identifying write amplification issues, detecting abnormally large write operations, and diagnosing I/O performance degradation in the key-value engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 206, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current ID allocation", "group": "Cluster", "used_for": "Used for monitoring the current maximum ID allocation value in PD's ID allocator. Helps track ID allocation progress and diagnose potential ID exhaustion issues or allocation anomalies in the cluster.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"idalloc\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 203, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Cluster ID", "group": "Cluster", "used_for": "Used for displaying the unique cluster identifier of the PD (Placement Driver) cluster. Essential for verifying cluster identity when managing multiple TiDB clusters or troubleshooting cross-cluster issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22257, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Table Count", "group": "Changefeed", "used_for": "Used for monitoring the total number of tables being replicated by each TiCDC changefeed. Helps identify changefeeds with excessive table counts that may impact replication performance and resource utilization.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanager_table_event_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 205, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Label distribution", "group": "Cluster", "used_for": "Used for monitoring the distribution of placement labels across the TiDB cluster managed by PD. Helps verify that data placement rules are correctly applied and diagnose issues with region scheduling, data locality, and cluster topology configuration.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (name)", "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 319, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Number of Segments", "group": "Memory trace", "used_for": "Used for monitoring the number of DeltaTree segments and memtables in TiFlash's storage engine. Essential for tracking memory usage patterns and diagnosing memory pressure issues caused by excessive segment fragmentation or memtable accumulation.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_DT_NumSegment{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_NumMemTable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 169, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Effectiveness of Mark Cache", "group": "Memory trace", "used_for": "Used for monitoring the hit and miss rates of TiFlash's Mark Cache, which stores metadata about data block marks. Essential for diagnosing memory efficiency issues and determining if the Mark Cache size needs adjustment to improve query performance by reducing disk I/O for metadata lookups.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_system_profile_event_MarkCacheMisses{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "max(tiflash_system_profile_event_MarkCacheHits{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 228, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Shards", "group": "Kv Engine", "used_for": "Used for monitoring the distribution and status of shards in TiKV's key-value storage engine, tracking total shards, initially flushed shards, and currently active shards. Essential for understanding data partitioning health and identifying potential shard management issues that could impact storage efficiency and query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_shards_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_initial_flushed_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_active_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_compacting_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_pending_compaction_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_has_del_prefixes_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_inner_key_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_ready_destroy_range_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 207, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "CPU Usage", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of PD (Placement Driver) components including PD, TSO, and scheduling services. Essential for identifying CPU bottlenecks, resource contention, and determining if PD nodes are overloaded, which can impact cluster metadata management and timestamp allocation performance.", "component": "PD", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 293, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "PD Leader History", "group": "Server", "used_for": "Used for tracking PD leader election history and identifying leadership changes in the cluster. Essential for diagnosing cluster stability issues, understanding when leader transfers occurred, and correlating service disruptions with PD leadership changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_tso_role{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22256, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Node Table Count", "group": "Changefeed", "used_for": "Used for monitoring the number of table event dispatchers per TiCDC node in a changefeed. Helps identify workload distribution imbalances across TiCDC instances and diagnose potential bottlenecks when certain nodes handle disproportionate table counts.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanager_table_event_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 86, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Checkpoint", "group": "Changefeed", "used_for": "Used for monitoring the replication progress of TiCDC changefeeds by comparing the changefeed checkpoint timestamp against the current PD cluster TSO. Essential for identifying replication lag, detecting stuck changefeeds, and ensuring data synchronization is keeping pace with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"})", "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 664, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Flush Rows / s", "group": "Dataflow", "used_for": "Used for monitoring the rate of rows being flushed to downstream sinks per second in TiCDC. Essential for tracking data replication throughput and identifying bottlenecks in the sink write pipeline when replication lag occurs.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 230, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Files Size", "group": "Kv Engine", "used_for": "Used for monitoring the total size of SST files across different column families and levels in TiKV's RocksDB storage engine. Essential for tracking storage space usage, identifying data distribution imbalances, and diagnosing compaction issues that may lead to disk space problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_cf_level_total_sizes_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 226, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Keys / Write", "group": "Kv Engine", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys written per write operation in TiKV's storage engine, showing p99, p9999 percentiles and average values. Essential for identifying write amplification issues, detecting abnormal batch sizes, and diagnosing performance problems when applications write unusually large or small key batches to TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 231, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memtable Count", "group": "Kv Engine", "used_for": "Used for monitoring the number of active memtables in TiKV's RocksDB storage engine. Essential for diagnosing memory pressure issues and understanding write buffer utilization, as excessive memtables can indicate slow flush operations or high write load.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_mem_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 337, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Snapshots", "group": "Memory trace", "used_for": "Used for monitoring TiFlash read snapshot activity and memory retention. Tracks active segment read tasks and the lifetime of the oldest delta snapshot, helping diagnose memory pressure issues caused by long-running queries or delayed snapshot cleanup in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 229, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Files Count", "group": "Kv Engine", "used_for": "Used for monitoring the number of SST files across different column families and levels in TiKV's RocksDB storage engine. Essential for diagnosing compaction issues, storage amplification problems, and detecting excessive file accumulation that can impact read performance and storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_cf_level_num_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 168, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Schema of Column File", "group": "Memory trace", "used_for": "Used for monitoring the schema cache of TiFlash's column file storage, tracking current cache size, hit rate, and eviction behavior per instance. Essential for diagnosing memory pressure issues and optimizing schema cache efficiency when TiFlash experiences high memory usage or performance degradation due to frequent schema lookups.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"current_size\"}) by (instance)", "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"hit_count\"}[2m])) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"still_used_when_evict\"}) by (instance)", "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"miss_count\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10037, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "GC Time", "group": "Changefeed", "used_for": "**Used For:**\n\nUsed for monitoring TiCDC's garbage collection (GC) safe points, tracking both the minimum service GC safe point and CDC's own GC safe point. Essential for diagnosing data retention issues, replication lag problems, and ensuring TiCDC doesn't block cluster-wide GC operations that could lead to storage bloat.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(ticdc_gc_min_service_gc_safepoint{})", "max(ticdc_gc_cdc_gc_safepoint{})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 209, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring how long each PD (Placement Driver) instance has been running since its last restart. Essential for tracking cluster stability, identifying recent restarts or crashes, and verifying that PD components remain continuously available without unexpected downtime.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 272, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Memory by thread (proxy)", "group": "Memory trace", "used_for": "Used for monitoring memory allocation and deallocation rates across different threads in TiFlash's Raft proxy component. Essential for diagnosing memory leaks, identifying threads with excessive memory consumption, and troubleshooting TiFlash proxy memory pressure issues.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "-rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}", "-tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 208, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Memory Usage", "group": "Cluster", "used_for": "Used for monitoring memory consumption of PD cluster components (PD, TSO, and Scheduling services), tracking both process resident memory and Go heap allocation. Essential for detecting memory leaks, identifying OOM risks, and diagnosing performance degradation caused by excessive memory usage in the cluster management layer.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}", "pd_service_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Checkpoint Lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC changefeed's checkpoint timestamp and upstream TiDB's current timestamp. Critical for identifying data synchronization delays and ensuring downstream systems receive near real-time updates from the source database.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 512, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Resolved Ts", "group": "Changefeed", "used_for": "Used for monitoring TiCDC changefeed's data replication progress by tracking resolved timestamps against the cluster's current TSO. Essential for identifying replication lag, detecting slow tables that are blocking progress, and diagnosing changefeed synchronization delays in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"})", "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 289, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Memory by thread", "group": "Memory trace", "used_for": "Used for monitoring memory allocation and deallocation rates across different threads in TiFlash instances. Essential for identifying memory leaks, tracking thread-level memory consumption patterns, and diagnosing out-of-memory issues in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "-rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}", "-tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 232, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memtable Size", "group": "Kv Engine", "used_for": "Used for monitoring the total memory consumed by memtables in TiKV's RocksDB key-value engine. Essential for diagnosing memory pressure issues and understanding write buffer usage, as large memtable sizes can indicate write-heavy workloads or delayed flush operations.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\",type=\"memtable\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 210, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Goroutine Count", "group": "Cluster", "used_for": "Used for monitoring the number of active goroutines in PD (Placement Driver) components including PD, TSO, and scheduling services. Helps identify goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate performance degradation or potential memory issues in the cluster's control plane.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 291, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Memory by class", "group": "Memory trace", "used_for": "Used for monitoring memory consumption breakdown by allocation class in TiFlash instances. Helps identify which memory categories (e.g., cache, query execution, storage) are consuming the most resources, essential for diagnosing memory pressure issues and optimizing TiFlash memory configuration.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "rate(tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__interval])" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 513, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Resolved Ts Lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays in changefeeds, helping diagnose slow table replication issues and ensuring near real-time data consistency in downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 398, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Checkpoint Lag", "group": "Lag analyze", "used_for": "Used for monitoring the replication lag between TiCDC changefeed's checkpoint timestamp and upstream TiDB. Critical for identifying data synchronization delays and ensuring downstream systems receive timely updates, especially important for real-time replication scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 233, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "L0 Table Count", "group": "Kv Engine", "used_for": "**Used For:**\n\nUsed for monitoring the number of SST files in Level 0 of TiKV's RocksDB storage engine. Critical for identifying compaction bottlenecks and write stalls, as excessive L0 files indicate that compaction cannot keep up with write throughput, potentially causing performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_l0_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 263, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "KVStore memory", "group": "Memory trace", "used_for": "Used for monitoring the memory consumption of TiFlash's KVStore component per instance. Essential for diagnosing memory pressure issues and identifying instances with abnormal storage engine memory usage that could lead to OOM problems.", "component": "TIFLASH", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingKVStore{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 468, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Resolved Ts Lag", "group": "Lag analyze", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays in changefeeds and ensuring near real-time data replication to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 234, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "L0 Table Size", "group": "Kv Engine", "used_for": "Used for monitoring the total size of L0 (Level 0) SST files in TiKV's RocksDB storage engine per instance. Critical for identifying compaction bottlenecks and write amplification issues, as excessive L0 table accumulation indicates the compaction process cannot keep up with write workload, potentially leading to performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_l0_tables_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 216, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory Usage", "group": "Kv Engine", "used_for": "**Used For:**\n\nUsed for monitoring memory consumption in TiKV's storage engine, including RocksDB components (memtables, block cache, bloom filters), DFS load memory, and index/data segment memory. Critical for diagnosing out-of-memory issues, identifying memory pressure, and optimizing TiKV memory configuration to prevent OOM crashes.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\"}\n \n)) by (type, $additional_groupby) ", "max((\n kv_engine_dfs_load_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 235, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Blob Table Count", "group": "Kv Engine", "used_for": "Used for monitoring the number of blob file tables in TiKV's Titan storage engine. Helps track blob file accumulation and diagnose potential issues with blob file management or garbage collection that could impact storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_blob_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 212, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Max TSO gap", "group": "Cluster", "used_for": "Used for monitoring the maximum TSO (Timestamp Oracle) gap in milliseconds between different data centers in a PD cluster. Critical for detecting clock synchronization issues and TSO allocation delays that could impact transaction consistency in multi-DC deployments.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 469, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Eventfeed Error / m", "group": "Lag analyze", "used_for": "Used for monitoring the rate of event feed errors per minute in TiCDC's KV client, categorized by error type. Essential for diagnosing data replication issues and identifying specific failure patterns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 466, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "PD Operator / m", "group": "Lag analyze", "used_for": "Used for monitoring the rate of PD scheduling operations created per minute, broken down by operation type. Helps diagnose TiCDC replication lag by identifying excessive region scheduling activity that may impact change data capture performance and stability.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 485, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Write Duration", "group": "Lag analyze", "used_for": "Used for monitoring the time TiCDC takes to write data to downstream sinks (like MySQL, Kafka, or storage systems). Critical for diagnosing replication lag issues and identifying bottlenecks in the data synchronization pipeline between upstream TiDB and downstream targets.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 211, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current TSO Physcial", "group": "Cluster", "used_for": "Used for monitoring the current physical timestamp component of PD's Timestamp Oracle (TSO). Essential for verifying TSO service health, detecting time synchronization issues, and diagnosing problems where TSO values appear frozen or regressing, which would block all transactions in the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 213, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Scheduling Allowance Status", "group": "Cluster", "used_for": "Used for monitoring whether PD's scheduling operations are currently allowed or restricted in the cluster. Essential for diagnosing why region scheduling might be paused or limited, helping identify if scheduling is intentionally disabled or blocked due to cluster conditions like high load or maintenance mode.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 236, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Blob Table Size", "group": "Kv Engine", "used_for": "Used for monitoring the blob file storage size in TiKV's storage engine, tracking both in-use and total blob data sizes per instance. Essential for understanding storage capacity utilization when using Titan engine's blob file separation feature and diagnosing storage space issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_in_use_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "sum((\n kv_engine_total_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 237, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Partial File Count", "group": "Kv Engine", "used_for": "Used for monitoring the count of partial files in TiKV's storage engine, including L0 files, blob files, and Ln-level files. Essential for diagnosing storage engine health and identifying potential compaction issues that could lead to increased read/write latency and degraded performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_partial_l0_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_partial_blob_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_partial_ln_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 479, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiDB Query Duration", "group": "Lag analyze", "used_for": "Used for monitoring the 99.9th percentile query duration in TiDB, excluding internal queries and simple commands. Critical for identifying tail latency issues in TiCDC replication scenarios and diagnosing slow queries that may contribute to replication lag.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[2m])) by (le, sql_type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 214, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Current TSO Datetime", "group": "Cluster", "used_for": "Used for displaying the current timestamp oracle (TSO) value in the PD cluster, which represents the logical clock used for distributed transaction ordering. Helps verify TSO generation is functioning correctly and can be used to detect time synchronization issues or TSO allocation problems that would impact transaction consistency.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\"})by(type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 473, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Min Resolved Ts Lag", "group": "Lag analyze", "used_for": "Used for monitoring the replication lag between TiKV's resolved timestamp and the current cluster TSO in TiCDC deployments. Critical for identifying data replication delays and diagnosing whether TiKV is falling behind in advancing resolved timestamps, which directly impacts CDC change capture latency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 221, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write operations", "group": "Kv Engine", "used_for": "Used for monitoring the rate of write operations per second in TiKV's key-value storage engine. Helps identify write workload patterns, detect sudden spikes in write activity, and diagnose performance issues related to write-heavy operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 486, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Write Rows / s", "group": "Lag analyze", "used_for": "Used for monitoring the rate of rows written per second by TiCDC sinks to downstream systems. Essential for tracking replication throughput and diagnosing data synchronization bottlenecks or performance degradation in CDC pipelines.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 41, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of write operations in TiFlash storage, tracking both storage command counts and DeltaTree write block events. Essential for identifying write bottlenecks, analyzing write workload patterns, and diagnosing performance issues when data ingestion from TiKV to TiFlash is slow or unstable.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (type)", "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 239, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Entries Count", "group": "Kv Engine", "used_for": "Used for monitoring the distribution of key-value entries in TiKV's storage engine, including total entries, old entries, and tombstones (deleted entries). Essential for diagnosing storage bloat, compaction efficiency issues, and identifying when garbage collection is needed to reclaim space from deleted data.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_old_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_tombstones\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 130, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the rate of large internal storage operations in TiFlash, including delta merge, segment merge, and segment split tasks. Essential for understanding TiFlash's background data reorganization activity and diagnosing storage layer performance or stability issues related to data compaction.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 258, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (bytes)", "group": "Storage", "used_for": "Used for monitoring the write throughput of TiFlash storage subtasks, broken down by operation type. Helps identify write performance bottlenecks and understand which storage operations are consuming the most bandwidth in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 240, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Top Write Shards ID", "group": "Kv Engine", "used_for": "Used for identifying the top write-intensive shards (data partitions) in TiKV's storage engine by their IDs and ranking. Helps diagnose write hotspot issues by revealing which specific shards are receiving disproportionate write traffic, enabling targeted load balancing and performance optimization.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_top_10_write_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",rank=\"1\"}\n \n)) by (instance) ", "sum((\n kv_engine_top_10_write_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (rank, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 39, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of small internal maintenance tasks in TiFlash storage engine, excluding major operations like delta merge, segment merge, and segment split. Helps identify background task activity levels and potential bottlenecks in routine storage maintenance operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 304, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operator timeout", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that timeout before completion, broken down by operator type. Critical for identifying scheduling bottlenecks, resource constraints, or configuration issues that prevent PD from successfully completing region rebalancing and scheduling operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"timeout\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 301, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operator create", "group": "Operator", "used_for": "Used for monitoring the rate at which PD creates different types of scheduling operators (e.g., region balance, leader transfer). Essential for understanding PD's scheduling activity and diagnosing whether the scheduler is actively responding to cluster imbalances or hotspots.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 303, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operator finish", "group": "Operator", "used_for": "Used for monitoring the rate at which PD scheduling operators complete their operations, broken down by operator type. Essential for tracking scheduling activity and identifying which types of region movements (leader transfers, region merges, etc.) are being executed most frequently in the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"finish\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 67, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Current Data Management Tasks", "group": "Storage", "used_for": "Used for monitoring active data management operations in TiFlash's DeltaTree storage engine, including delta-merge compactions, segment splits, and segment merges. Essential for diagnosing TiFlash storage performance issues and understanding background task activity that affects query latency and resource utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 259, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (rows)", "group": "Storage", "used_for": "Used for monitoring the row-level write throughput of TiFlash storage subtasks, broken down by operation type. Essential for analyzing TiFlash's data ingestion performance and identifying bottlenecks in different write operations when troubleshooting slow data synchronization or replication issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 242, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Level Write Throughput", "group": "Kv Engine", "used_for": "Used for monitoring the write throughput across different LSM-tree levels in TiKV's RocksDB engine. Helps diagnose write amplification issues and compaction performance by showing how data flows through storage levels during write operations.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_level_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 305, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operator replaced or canceled", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that are canceled or replaced, broken down by operator type. Helps diagnose scheduling instability, conflicts between operators, or frequent scheduling adjustments that may indicate cluster balance issues or configuration problems.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"cancel\"}[2m])*60) by (type)", "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"replace\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 302, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operator check", "group": "Operator", "used_for": "Used for monitoring the rate at which PD checks different types of scheduling operators per minute. Helps diagnose scheduling decision-making activity and identify when PD is actively evaluating potential region movements or balance operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"check\"}[2m])*60) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 38, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Amplification", "group": "Storage", "used_for": "Used for monitoring the write amplification ratio in TiFlash storage, which measures how much actual disk I/O is performed compared to logical writes. Critical for identifying inefficient storage patterns, excessive compaction overhead, and diagnosing performance degradation caused by high write amplification in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[10m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[30m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m])\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 42, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the P99 and P9999 latency of internal storage subtasks in TiFlash, such as data compaction, delta merge, and background maintenance operations. Critical for diagnosing TiFlash storage layer performance issues and identifying when internal tasks are causing delays in query processing or data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type, $additional_groupby) / 1000000000)", "histogram_quantile(0.9999, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 401, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Scan Tasks / m", "group": "Lag analyze", "used_for": "Used for monitoring the rate of TiKV CDC scan tasks per minute, broken down by scan type and instance. Helps diagnose TiCDC replication lag by identifying which TiKV nodes are performing excessive scanning operations or experiencing scan bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 306, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedule operators count by state", "group": "Operator", "used_for": "Used for monitoring the rate of scheduling operator state changes in PD, categorized by event type (created, finished, timeout, etc.). Essential for diagnosing scheduling behavior, identifying stuck operators, and understanding whether the cluster is actively rebalancing regions or experiencing scheduling bottlenecks.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 244, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Request Retry Rate", "group": "Kv Engine", "used_for": "Used for monitoring the rate of retry attempts in TiKV's distributed file system (DFS) operations. Helps identify storage layer instability, network issues, or resource contention that causes DFS requests to fail and require retries.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_dfs_retry_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 241, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Throughput", "group": "Kv Engine", "used_for": "Used for monitoring the throughput of TiKV's Distributed File System (DFS) operations, specifically S3 storage interactions. Essential for diagnosing performance bottlenecks in cloud storage backends and validating data transfer rates when using remote storage solutions.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_dfs_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",dfs_type=\"s3\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 243, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Request Rate", "group": "Kv Engine", "used_for": "Used for monitoring the rate of Distributed File System (DFS) requests in TiKV's storage engine. Helps identify DFS access patterns and diagnose issues related to remote storage operations, particularly useful when TiKV is configured with disaggregated storage architecture.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_dfs_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 131, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks Duration", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring the p99 and p9999 latency of large internal storage tasks in TiFlash, such as data compaction and delta merge operations. Critical for diagnosing TiFlash storage layer performance issues and identifying when background maintenance tasks are causing excessive delays that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type, $additional_groupby) / 1000000000)", "histogram_quantile(0.9999, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 407, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Leader Change", "group": "Lag analyze", "used_for": "Used for monitoring the rate of Raft leader region changes in TiKV instances over 2-minute intervals. Essential for diagnosing replication lag issues in TiCDC, as frequent leader transfers can cause data capture delays and indicate cluster instability or load balancing problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 467, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Scan Region Time / m", "group": "Lag analyze", "used_for": "Used for monitoring the total time TiKV spends scanning regions for CDC (Change Data Capture) operations per minute. Essential for diagnosing TiCDC replication lag caused by slow region scanning, helping identify I/O bottlenecks or inefficient scan operations in the source TiKV nodes.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "File Open OPS", "group": "Storage", "used_for": "Used for monitoring the rate of file open operations and failures in TiFlash storage layer. Essential for diagnosing I/O bottlenecks, file descriptor exhaustion, or storage access issues that could impact TiFlash query performance and stability.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 308, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Operator step finish duration", "group": "Operator", "used_for": "Used for monitoring the p95 and p99 latency of PD operator step completion during scheduling operations. Critical for diagnosing scheduling performance issues and identifying slow operator execution that may delay region rebalancing, leader transfers, or other cluster management tasks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 307, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Operator finish duration", "group": "Operator", "used_for": "Used for monitoring the P95 and P99 completion time of PD scheduling operators by type. Essential for diagnosing slow operator execution and identifying which operator types (e.g., region merge, leader transfer) are experiencing latency issues that could delay cluster rebalancing and scheduling operations.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 310, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% operator region size", "group": "Operator", "used_for": "Used for monitoring the 99th percentile size of regions involved in PD scheduling operations. Helps identify whether large regions are causing scheduling bottlenecks or uneven data distribution issues across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type,le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 401, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store capacity", "group": "Statistics - balance", "used_for": "Used for monitoring the total storage capacity of each TiKV store in the cluster. Essential for capacity planning and identifying stores approaching their storage limits before they impact cluster availability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_capacity\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 50, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Opened File Count", "group": "Storage", "used_for": "Used for monitoring the number of open file descriptors in TiFlash instances, including proxy process files, files opened for writing, and files opened for reading. Critical for detecting file descriptor leaks or approaching system limits that could cause TiFlash failures or performance degradation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}", "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 309, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Schedulers reach limit", "group": "Operator", "used_for": "Used for monitoring when PD schedulers hit their configured rate limits, tracking both operator limits by type/name and store-level limit violations. Essential for diagnosing why region scheduling operations are being throttled or delayed, helping identify if limits need adjustment during cluster rebalancing or recovery scenarios.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (type,name)", "sum(rate(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (desc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 471, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Advance Resolved Ts / s", "group": "Lag analyze", "used_for": "Used for monitoring the rate at which TiKV advances resolved timestamps per second for CDC (Change Data Capture) operations. Helps diagnose CDC replication lag issues by showing how frequently TiKV updates the resolved timestamp, which is critical for identifying bottlenecks in change data synchronization performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) " ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 400, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Unresolved Region Count", "group": "Lag analyze", "used_for": "Used for monitoring the number of unresolved regions in TiKV during CDC (Change Data Capture) replication. Critical for diagnosing CDC lag issues, as unresolved regions indicate pending data changes that haven't been captured yet, directly impacting replication latency and data consistency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 52, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FSync Status", "group": "Storage", "used_for": "Used for monitoring the frequency and latency of fsync operations in TiFlash storage. Critical for diagnosing disk I/O performance issues and identifying instances experiencing slow data persistence, which can impact write durability and overall TiFlash stability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 246, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Request Duration with Retry", "group": "Kv Engine", "used_for": "Used for monitoring the latency of Disaggregated File System (DFS) requests in TiKV, including retry attempts, at p99 and p9999 percentiles. Critical for diagnosing performance issues in disaggregated storage architectures where compute and storage are separated, helping identify slow remote storage access or retry storms.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 470, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Admin Apply / s", "group": "Lag analyze", "used_for": "Used for monitoring the rate of successful TiKV admin commands (excluding compaction) per second, broken down by command type. Essential for analyzing TiCDC replication lag by identifying admin operation bottlenecks like region splits, merges, or configuration changes that may impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 402, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store available", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the number of available stores in the TiKV cluster that can accept scheduling operations. Essential for diagnosing cluster capacity issues, identifying unavailable stores, and understanding why PD may not be performing region balancing or scheduling tasks.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 248, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Prepare ChangeSet File Load Source", "group": "Kv Engine", "used_for": "Used for monitoring the rate of changeset file loading operations in TiKV's key-value engine, distinguishing between remote file loads and local file usage. Helps diagnose performance issues related to data ingestion and identify whether the system is efficiently utilizing local versus remote file sources during changeset preparation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_prepare_load_remote_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n kv_engine_prepare_use_local_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Disk Write OPS", "group": "Storage", "used_for": "Used for monitoring disk write operations per second in TiFlash storage, including PSM write calls, page writes, and file descriptor writes. Essential for diagnosing write throughput bottlenecks, identifying storage performance degradation, and correlating disk I/O patterns with query workload changes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 110, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Ownership History", "group": "Server", "used_for": "Used for tracking ownership changes in TiCDC's distributed coordination system. Helps identify frequent leader elections or ownership transfers that may indicate cluster instability or coordination issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"}[240s])) by (instance) > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 433, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Check Leader Region Count Percentile", "group": "Lag analyze", "used_for": "Used for monitoring the 99.99th percentile of region count per check leader request in TiKV instances. Critical for diagnosing TiCDC replication lag issues caused by excessive region scanning, helping identify when TiKV needs to check too many regions during leader verification operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 110, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Online store progress", "group": "", "used_for": "Used for monitoring the progress of store preparation operations in PD, specifically tracking stores that are in the \"preparing\" state during online operations. Helps administrators understand the status of store initialization or migration processes and identify if stores are stuck in the preparation phase.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"preparing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL schema changes being applied to TiFlash replicas at various percentiles (p80-p999). Critical for diagnosing slow DDL propagation issues and identifying when schema synchronization between TiDB and TiFlash becomes a bottleneck during table alterations or schema updates.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role", "type" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", keyspace!=\"\"}[2m])) by (le, type, keyspace))", "sum(tiflash_sync_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"$type\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 247, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Load Memory Wait Duration", "group": "Kv Engine", "used_for": "Used for monitoring the wait duration when TiKV's Disaggregated File System (DFS) loads data into memory, tracking both p99 and p9999 latencies. Critical for diagnosing memory pressure issues and identifying when remote storage reads are blocked waiting for available memory buffers in disaggregated storage architectures.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 404, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store available ratio", "group": "Statistics - balance", "used_for": "Used for monitoring the available storage capacity ratio of each TiKV store in the cluster. Essential for identifying stores approaching capacity limits and diagnosing unbalanced storage distribution that could trigger region rebalancing or cause write failures.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_capacity\"}) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 47, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Disk Read OPS", "group": "Storage", "used_for": "Used for monitoring the rate of disk read operations in TiFlash storage, including PSM read I/O calls, page reads, and file descriptor reads. Essential for diagnosing storage I/O bottlenecks, identifying excessive disk activity, and correlating read throughput with query performance issues in TiFlash analytical workloads.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 249, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CacheFS Hit", "group": "Kv Engine", "used_for": "Used for monitoring the cache hit rate of TiKV's CacheFS feature, which caches frequently accessed data in faster storage layers. Essential for evaluating cache effectiveness and identifying opportunities to improve read performance by optimizing cache configuration or capacity.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 427, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Advance Resolved Ts Fail / m", "group": "Lag analyze", "used_for": "Used for monitoring the rate of failures when TiKV attempts to advance the resolved timestamp for CDC replication. Critical for diagnosing TiCDC replication lag issues by identifying which TiKV instances are failing to advance resolved timestamps and the specific failure reasons causing replication delays.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 59, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read flow", "group": "Storage", "used_for": "Used for monitoring the read I/O throughput in TiFlash storage layer, including file descriptor reads, PageStorage (PSM) reads, and background reads. Essential for diagnosing read performance bottlenecks, identifying heavy read workloads, and understanding TiFlash's storage I/O patterns during analytical queries.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 472, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV Check Leader Duration Percentile", "group": "Lag analyze", "used_for": "Used for monitoring the 99.99th percentile latency of TiKV's check leader operations in TiCDC's resolved timestamp mechanism. Critical for diagnosing replication lag issues caused by slow leader verification, helping identify when TiKV takes excessive time to confirm region leadership during change data capture.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 88, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Snapshots", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring active read snapshots and MVCC snapshot management in TiFlash's storage layer. Helps diagnose read concurrency issues, snapshot accumulation problems, and potential memory pressure from long-running queries holding snapshots.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 60, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write flow", "group": "Storage", "used_for": "Used for monitoring the write throughput in TiFlash storage layer, tracking bytes written through file descriptors, PageStorage Manager (PSM) foreground writes, and PSM background writes. Essential for diagnosing write performance bottlenecks, understanding I/O patterns, and identifying whether write pressure comes from user queries or background compaction tasks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 406, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store Region score", "group": "Statistics - balance", "used_for": "Used for monitoring the region distribution score across TiKV stores and the balance-region-scheduler's behavior. Essential for diagnosing uneven region distribution issues, understanding why PD schedules region transfers, and identifying stores that are overloaded or underutilized in terms of region count.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_score\"}", "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}", "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 40, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Tasks OPS", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the rate of read tasks processed per second by each TiFlash instance's storage read pool. Essential for identifying read throughput bottlenecks, uneven load distribution across TiFlash nodes, and diagnosing performance issues in analytical query processing.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 403, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store used", "group": "Statistics - balance", "used_for": "Used for monitoring the actual disk space consumed by each TiKV store in the cluster. Essential for tracking storage utilization trends, identifying stores approaching capacity limits, and diagnosing unbalanced data distribution issues that may require rebalancing.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_used\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 253, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA Segment Memory Cache", "group": "Kv Engine", "used_for": "Used for monitoring memory consumption of the In-Memory Aggregation (IA) segment cache in TiKV's storage engine. Essential for tracking memory usage patterns and diagnosing potential memory pressure issues related to the IA manager's segment storage and small queue capacity.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "max((\n kv_engine_ia_small_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 408, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store Region size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of regions stored on each TiKV store in the cluster. Essential for identifying storage imbalance issues and determining if region distribution needs rebalancing across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22445, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Coordinator Operator Cost Duration", "group": "Coordinator", "used_for": "Used for monitoring the 99.9th percentile latency of coordinator operator operations in TiCDC's new architecture. Critical for identifying performance bottlenecks in change data capture coordination tasks, such as scheduling table dispatching and rebalancing operations across capture nodes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_coordinator_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,mode))", "sum(rate(ticdc_coordinator_finish_operators_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,mode) / \nsum(rate(ticdc_coordinator_finish_operators_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22444, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Status", "group": "Coordinator", "used_for": "Used for monitoring the operational state of TiCDC changefeeds across different keyspaces and namespaces. Essential for quickly identifying changefeed health issues, tracking state transitions (normal, stopped, error), and diagnosing replication problems in the TiCDC coordinator component.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_coordinator_changefeed_state{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed, state)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 269, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Scheduling", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring TiFlash read thread scheduling efficiency and resource utilization. Helps diagnose read performance bottlenecks caused by thread exhaustion, scheduling limits, or resource contention in the storage read pool.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 113, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Placement Rules Status", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the status and health of PD's placement rules manager, which controls data replica placement across the cluster. Essential for diagnosing issues with data distribution policies, replica placement failures, and ensuring placement rules are functioning correctly to maintain data availability and compliance with configured constraints.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_rule_manager_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 405, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Size amplification", "group": "Statistics - balance", "used_for": "Used for monitoring the storage amplification ratio in TiKV stores by comparing actual region size to available storage capacity. Helps identify stores with inefficient space utilization, excessive data fragmentation, or compaction issues that may require rebalancing or maintenance.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_used\"}) by (address, store) * 2^20" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 407, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store leader score", "group": "Statistics - balance", "used_for": "Used for monitoring the leader score of each TiKV store, which indicates the distribution balance of Region leaders across stores. Essential for diagnosing uneven leader distribution issues and verifying whether PD's leader balancing scheduler is working effectively to distribute load evenly across the cluster.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_score\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 250, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CacheFS Hit Rate", "group": "Kv Engine", "used_for": "Used for monitoring the cache hit rate of TiKV's CacheFS layer, showing the percentage of requests served from cache versus total requests. Essential for evaluating cache effectiveness and identifying opportunities to improve read performance by optimizing cache configuration or capacity.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10038, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "TiKV CDC Incremental Scan Long Duration Region Count", "group": "Lag analyze", "used_for": "Used for monitoring the number of TiKV regions experiencing prolonged incremental scan durations during CDC (Change Data Capture) operations. Critical for diagnosing CDC replication lag caused by slow region scanning, helping identify performance bottlenecks in change data synchronization.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "tikv_cdc_scan_long_duration_region{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 256, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA Segment Cache Hit Rate", "group": "Kv Engine", "used_for": "Used for monitoring the cache hit rate of the Index Advisor (IA) segment cache in TiKV's storage engine. Helps identify cache efficiency issues and determine if the segment cache size needs adjustment to improve read performance and reduce disk I/O overhead.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(1 - (sum(rate(\n kv_engine_ia_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_ia_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Ingest Level Rate", "group": "Kv Engine", "used_for": "Used for monitoring the rate of SST file ingestion operations across different LSM-tree levels in TiKV's RocksDB engine. Helps diagnose write amplification issues and understand data compaction patterns, particularly useful when investigating storage performance degradation or unusual disk I/O behavior.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_ingest_level_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 245, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "DFS Request Duration", "group": "Kv Engine", "used_for": "Used for monitoring the latency of Distributed File System (DFS) requests in TiKV's storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing storage I/O performance issues and identifying tail latency problems when TiKV interacts with distributed file systems or remote storage backends.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_dfs_latency_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 409, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store leader size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of Raft leader regions on each TiKV store. Essential for diagnosing leader distribution imbalances that can cause hotspots and uneven load across the cluster, helping identify when leader rebalancing is needed.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22446, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Coordinator History", "group": "Coordinator", "used_for": "Used for tracking TiCDC coordinator ownership changes and failovers over time. Helps identify coordinator stability issues, frequent leadership transfers, or split-brain scenarios that could impact change data capture reliability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"}[240s])) by (instance) > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 257, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA Segmenet Read Duration", "group": "Kv Engine", "used_for": "Used for monitoring the read latency of IA (In-memory Archive) segments in TiKV's storage engine. Critical for diagnosing performance issues when accessing archived data and identifying bottlenecks in the IA segment read path.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n kv_engine_ia_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 255, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA Segment Cache Hit", "group": "Kv Engine", "used_for": "Used for monitoring the cache hit rate of IA (In-memory Archive) segments in TiKV's storage engine. Helps evaluate memory cache effectiveness and identify opportunities to optimize read performance by reducing disk I/O for frequently accessed archived data.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n kv_engine_ia_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) - sum(rate(\n kv_engine_ia_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 410, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store Region count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of regions stored on each TiKV store in the cluster. Essential for identifying region distribution imbalances that can lead to hotspots, uneven load distribution, and diagnosing whether PD's region scheduling is working effectively to balance data across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10069, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Changefeed Maintainer Count", "group": "Maintainer", "used_for": "Used for monitoring the number of active changefeed maintainers in TiCDC's new architecture. Helps verify proper changefeed distribution across TiCDC nodes and diagnose issues when changefeeds fail to start or are unexpectedly stopped.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_changefeed_maintainer_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 292, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Internal Duration", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the internal processing latency of TiFlash read threads at various percentiles (p80-p999). Critical for diagnosing TiFlash read performance bottlenecks and identifying whether delays occur within the read thread processing itself, helping distinguish internal processing issues from external factors like network or storage I/O.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 132, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Data Sharing", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring TiFlash's data sharing efficiency and cache hit rates during read operations. Helps diagnose whether the column cache is effectively sharing data between concurrent queries, reducing redundant I/O and improving read performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"add_cache_total_bytes_limit\"}[2m])) by (type)", "sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit\"}[2m]))/sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit|data_sharing_miss\"}[2m]))", "sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit\"}[2m]))/sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit|extra_column_miss\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 412, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store leader count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of Raft leaders distributed across each TiKV store in the cluster. Essential for diagnosing leader distribution imbalances that can cause hotspots and uneven load, helping identify when PD's leader balancing scheduler needs adjustment.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 258, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% IA segment read duration", "group": "Kv Engine", "used_for": "Used for monitoring the 99th and 99.99th percentile read latency of IA (In-memory Accelerator) segments in TiKV's storage engine. Critical for diagnosing performance issues with TiKV's in-memory cache layer and identifying tail latency problems that affect read-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_ia_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_ia_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_ia_read_segment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_ia_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n kv_engine_ia_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10071, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Maintainer Handle Event Duration", "group": "Maintainer", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC maintainer processing change events. Critical for diagnosing replication lag issues and identifying performance bottlenecks in the CDC event handling pipeline, especially when changefeeds experience delays in capturing and processing database changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_maintainer_handle_event_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed))", "sum(rate(ticdc_maintainer_handle_event_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed) / \nsum(rate(ticdc_maintainer_handle_event_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 254, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA Segment Disk Cache", "group": "Kv Engine", "used_for": "Used for monitoring the disk cache usage of the In-Memory Accelerator (IA) segment storage in TiKV, comparing actual disk size against the configured capacity. Essential for tracking cache utilization and identifying when the IA disk cache is approaching its limits, which could impact query acceleration performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n kv_engine_ia_manager_segments_disk_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "max((\n kv_engine_ia_main_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 362, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Segment MergedTask Duration", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the latency distribution (p80, p99, p999) of segment merged tasks in TiFlash's storage read pool. Critical for diagnosing read performance issues and identifying whether data sharing operations are experiencing delays that could impact query response times.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_storage_read_thread_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type,$additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_read_thread_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type,$additional_groupby))", "histogram_quantile(0.80, sum(rate(tiflash_storage_read_thread_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type,$additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 361, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Segment MergedTask", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the distribution of merged read tasks in TiFlash's storage read thread pool by task type. Helps diagnose read I/O efficiency and identify whether data sharing mechanisms are effectively consolidating similar read requests to reduce redundant disk operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (type,$additional_groupby) (tiflash_storage_read_thread_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 501, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Hot Region's leader distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write region leaders across TiKV stores. Helps identify unbalanced hot region leadership that can cause write hotspots and performance bottlenecks on specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_leader\"}", "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",name=~\".*threshold\",type=~\"write.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 294, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Compression Ratio", "group": "Storage", "used_for": "Used for monitoring the compression efficiency of TiFlash's columnar storage, comparing uncompressed data size to compressed size for both LZ4 and lightweight compression algorithms. Essential for evaluating storage space utilization and identifying potential compression issues that could lead to excessive disk usage.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_compressed_bytes\"}[2m]))", "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_compressed_bytes\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 502, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write regions across TiKV stores from a peer perspective. Helps identify uneven hot region distribution and potential write hotspot imbalances that could lead to performance bottlenecks or storage node overload.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_peer\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20145, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Maintainer Checkpoint Lag", "group": "Maintainer", "used_for": "Used for monitoring the replication lag between TiCDC maintainer's checkpoint timestamp and the current time. Critical for identifying data synchronization delays in change data capture pipelines and ensuring timely downstream data availability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_maintainer_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 259, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage Size by Storage Class", "group": "Kv Engine", "used_for": "Used for monitoring the distribution of storage space consumption across different storage classes in TiKV. Helps identify storage capacity usage patterns and potential imbalances when using multiple storage tiers or classes in Kubernetes environments.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"used\"}\n \n)) by (storage_class, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12050, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Input Events / s", "group": "Log Puller", "used_for": "Used for monitoring the rate of events pulled from TiKV by TiCDC's log puller component, broken down by instance and event type. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 237, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexError", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the rate of delta index errors in TiFlash's DeltaTree storage engine. Critical for identifying data consistency issues or corruption in TiFlash's columnar storage layer that could impact query accuracy.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22269, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Region Request Finish Scan Duration", "group": "Log Puller", "used_for": "Used for monitoring the p99 latency of TiCDC's region request scan completion operations in the log puller component. Critical for diagnosing slow data capture issues and identifying performance bottlenecks when TiCDC scans regions to pull change data from TiKV.", "component": "TICDC", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_subscription_client_region_request_finish_scan_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "sum(rate(ticdc_subscription_client_region_request_finish_scan_duration_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)\n/\nsum(rate(ticdc_subscription_client_region_request_finish_scan_duration_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 301, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "VersionChain", "group": "Storage Read Pool & Data Sharing", "used_for": "Used for monitoring the time spent traversing MVCC version chains during TiFlash read operations at various percentiles (p80-p999). Critical for diagnosing read performance degradation caused by long version chains, which typically indicates excessive updates/deletes or delayed garbage collection in the storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 504, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Total written keys on hot leader Regions", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the total number of keys written per second on hot leader Regions in the cluster. Essential for identifying write hotspots, diagnosing unbalanced write load distribution across stores, and detecting potential performance bottlenecks caused by concentrated write traffic on specific leader Regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_keys_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 503, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store write rate bytes", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring write throughput (bytes per second) across TiKV stores to identify hot write regions and unbalanced write loads. Essential for diagnosing write hotspots, evaluating store-level write distribution, and determining if PD's load balancing is effectively spreading write traffic across the cluster.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22291, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Memory Quota", "group": "Log Puller", "used_for": "Used for monitoring memory quota usage in TiCDC's log puller module, broken down by instance and type. Essential for detecting memory pressure issues and potential OOM risks in the change data capture pipeline's log pulling component.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22296, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Unresolved Region Request Count ", "group": "Log Puller", "used_for": "Used for monitoring the number of region change event requests that TiCDC's log puller has sent but not yet received responses for. Critical for diagnosing replication lag and identifying potential bottlenecks in region data subscription, especially when requests accumulate due to TiKV overload or network issues.", "component": "TICDC", "phenomenon": "io", "object": "request", "percentile": [], "variables": [], "promql": [ "ticdc_subscription_client_requested_region_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 293, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Compression Algorithm Count", "group": "Storage", "used_for": "Used for monitoring the distribution of compression algorithms used by TiFlash storage engine when compressing data packs. Helps identify compression strategy patterns and diagnose storage efficiency issues related to data compression choices.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_pack_compression_algorithm_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 263, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Coprocessor Handle duration", "group": "In Memory Engine", "used_for": "**Used For:**\n\nUsed for monitoring the latency of coprocessor request handling in TiKV's In-Memory Engine, tracking p99 and p9999 percentiles. Critical for diagnosing slow analytical queries and identifying performance bottlenecks in coprocessor operations that process pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 261, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "OPS", "group": "In Memory Engine", "used_for": "Used for monitoring the operations per second (OPS) rate of TiKV's in-memory engine across key-value operations. Helps identify throughput patterns and detect performance bottlenecks in the in-memory storage layer when diagnosing read/write performance issues.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_kv_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 128, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Disk Usage", "group": "PageStorage", "used_for": "Used for monitoring TiFlash PageStorage disk space consumption, including total blob disk bytes, valid data bytes, and space utilization ratio. Essential for tracking storage efficiency, identifying space waste from garbage data, and preventing disk space exhaustion in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "sum(tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})) by (instance)", "tiflash_system_asynchronous_metric_LogDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22299, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Subscribed Region Count", "group": "Log Puller", "used_for": "**Used For:**\n\nUsed for monitoring the number of regions currently subscribed by TiCDC's log puller component. Essential for tracking data replication coverage and diagnosing issues when region subscriptions are incomplete or experiencing unexpected changes during change data capture operations.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "ticdc_subscription_client_subscribed_region_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 505, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Total written bytes on hot leader Regions", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the total write throughput (in bytes) on hot leader Regions across stores in the PD cluster. Essential for identifying write hotspots, detecting unbalanced write loads, and diagnosing performance bottlenecks caused by concentrated write traffic on specific leader Regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_bytes_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 264, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Cache Hit", "group": "In Memory Engine", "used_for": "**Used For:**\n\nUsed for monitoring the cache hit rate of regions stored in TiKV's In-Memory Engine. Helps evaluate memory cache effectiveness and identify whether frequently accessed regions are being served from memory, which is critical for optimizing read performance and determining if the in-memory engine configuration needs adjustment.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PageStorage WriteBatch Size", "group": "PageStorage", "used_for": "Used for monitoring the distribution of write batch sizes in TiFlash's PageStorage v3 engine. Helps diagnose write efficiency and identify whether data is being written in optimal batch sizes, which impacts I/O performance and storage throughput.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_storage_page_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"v3\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 265, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Cache Hit Rate", "group": "In Memory Engine", "used_for": "Used for monitoring the percentage of region data requests served from TiKV's in-memory engine cache versus disk storage. Critical for evaluating memory cache effectiveness and identifying opportunities to improve read performance by increasing cache hit rates.", "component": "TIKV", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 163, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Page GC Tasks OPM", "group": "PageStorage", "used_for": "Used for monitoring the rate of PageStorage garbage collection tasks in TiFlash, measured in operations per minute and categorized by GC type. Helps identify GC activity patterns and potential issues with storage space reclamation or excessive GC overhead impacting TiFlash performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 201, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "PD scheduler config", "group": "Cluster", "used_for": "Used for monitoring the configuration status of PD schedulers in the cluster. Helps verify scheduler configuration changes and diagnose scheduling behavior issues by tracking which scheduler settings are currently active.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 266, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Cache Miss Reason", "group": "In Memory Engine", "used_for": "Used for tracking the reasons why region cache lookups fail in TiKV's In-Memory Engine. Helps diagnose memory engine cache efficiency issues and identify patterns in cache misses, which can impact read performance when data needs to be fetched from disk instead of memory.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 267, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory Usage", "group": "In Memory Engine", "used_for": "Used for monitoring memory consumption of TiKV's In-Memory Engine across cluster instances. Essential for tracking memory usage patterns, detecting memory leaks, and ensuring the in-memory engine stays within allocated resource limits to prevent OOM issues.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_in_memory_engine_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Page GC Duration", "group": "PageStorage", "used_for": "Used for monitoring the 99th percentile and maximum duration of garbage collection operations in TiFlash's PageStorage engine. Critical for diagnosing storage performance degradation and identifying when GC pauses are impacting query latency or causing write stalls.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12046, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Resolved Ts Batch Size (Regions)", "group": "Log Puller", "used_for": "Used for monitoring the distribution of resolved timestamp batch sizes (measured in number of regions) in TiCDC's log puller component. Helps diagnose I/O efficiency and batching behavior when TiCDC processes region-level resolved timestamps, which is critical for identifying potential bottlenecks in change data capture throughput.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 20146, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Maintainer Resolved Ts Lag", "group": "Maintainer", "used_for": "Used for monitoring the replication lag between the maintainer's resolved timestamp and current time in TiCDC. Critical for identifying data synchronization delays and ensuring change data capture keeps pace with upstream TiDB transactions.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_maintainer_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 507, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store write query", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the write query rate at the store level in PD's hot region scheduling. Helps identify stores experiencing high write traffic and diagnose unbalanced write load distribution across TiKV nodes, which is critical for detecting hotspot issues and evaluating whether write load balancing is working effectively.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 269, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC Filter", "group": "In Memory Engine", "used_for": "Used for monitoring the rate of garbage collection filtering operations in TiKV's in-memory engine. Helps diagnose memory management efficiency and identify potential issues with stale data cleanup in the in-memory storage layer.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_gc_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Page write Duration", "group": "PageStorage", "used_for": "Used for monitoring write latency distribution (p80, p95, p99, p999) in TiFlash's PageStorage engine. Critical for diagnosing TiFlash write performance bottlenecks and identifying when data persistence operations are experiencing slowdowns or tail latency issues.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 509, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Selector write events", "group": "Statistics - hot write", "used_for": "Used for monitoring the rate of hot write region selection events in PD's scheduler. Helps diagnose hot write region detection and load balancing behavior, particularly when investigating uneven write distribution or hotspot issues across TiKV stores.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 129, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PageStorage File Num", "group": "PageStorage", "used_for": "Used for monitoring the number of blob files and log files in TiFlash's PageStorage engine per instance. Essential for tracking storage file accumulation and identifying potential issues with file compaction or cleanup processes that could impact storage efficiency and performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_asynchronous_metric_BlobFileNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_asynchronous_metric_LogNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 270, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region GC Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the garbage collection duration for regions in TiKV's in-memory engine. Critical for diagnosing memory management performance issues and identifying when GC operations are taking too long, which could impact overall cluster performance and memory utilization.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_gc_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 268, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Count", "group": "In Memory Engine", "used_for": "Used for monitoring the number of regions cached in TiKV's in-memory engine across instances. Helps identify memory usage patterns and potential issues with region cache distribution, which can impact read performance and memory pressure.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_in_memory_engine_cache_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22304, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Register Dispatcher StartTs Lag", "group": "Event Store", "used_for": "Used for monitoring the timestamp lag when TiCDC registers new dispatchers in the event store, showing p80, p95, and maximum latencies. Critical for diagnosing replication delays and identifying when change data capture falls behind the source database's transaction timeline.", "component": "TICDC", "phenomenon": "latency", "object": "store", "percentile": [ 80, 95 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.80, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 506, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store write rate keys", "group": "Statistics - hot write", "used_for": "Used for monitoring the write throughput in keys per second for each TiKV store, including both individual store rates and aggregated region-level write rates. Essential for identifying hot write stores, detecting write skew across the cluster, and diagnosing unbalanced write load distribution that could lead to performance bottlenecks.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22452, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Subscriptions Resolved Ts Lag", "group": "Event Store", "used_for": "Used for monitoring the lag between subscription resolved timestamps and current time in TiCDC's event store at p95, p80, and max percentiles. Critical for identifying replication delays and diagnosing whether TiCDC subscriptions are keeping up with upstream data changes in real-time.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [ 95 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.8, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 508, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Hot cache write entry numbers", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of hot write regions cached in PD's hotspot scheduler. Essential for diagnosing write hotspot detection accuracy and understanding how many regions PD identifies as having concentrated write traffic, which helps troubleshoot load balancing and hotspot scheduling issues.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 164, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Numer of Pages", "group": "PageStorage", "used_for": "Used for monitoring the number of data pages in TiFlash's PageStorage engine, including both in-memory pages and versioned entries. Essential for tracking memory usage patterns and diagnosing storage layer performance issues or memory pressure in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_system_asynchronous_metric_PagesInMem{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}", "tiflash_system_asynchronous_metric_VersionedEntries{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 510, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot write", "used_for": "Used for tracking the direction and frequency of hot write region leader transfers across TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing write-intensive leaders and identify stores that are frequently transferring leaders in or out due to write hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 231, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Pending Writers Num", "group": "PageStorage", "used_for": "Used for monitoring the number of pending write operations in TiFlash's PageStorage component per instance. Helps identify write bottlenecks and I/O congestion when pending writers accumulate, indicating potential disk performance issues or write pressure.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_PSPendingWriterNum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 271, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Load Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time taken to load regions into TiKV's in-memory engine. Critical for diagnosing performance issues related to region data loading and identifying bottlenecks in the in-memory engine's initialization process.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_load_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20022, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Resolved Ts Lag", "group": "Event Store", "used_for": "Used for monitoring the lag between the current time and the resolved timestamp in TiCDC's event store. Critical for identifying replication delays and ensuring data consistency in change data capture pipelines, as increasing lag indicates the event store is falling behind in processing upstream changes.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_event_store_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 262, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read MBps", "group": "In Memory Engine", "used_for": "Used for monitoring the read throughput (in megabytes per second) of TiKV's in-memory engine compared to the regular storage engine. Helps identify whether read operations are being served from memory cache versus disk, which is critical for diagnosing memory engine effectiveness and read performance optimization.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 198, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PageStorage stored bytes by type", "group": "PageStorage", "used_for": "Used for monitoring the distribution of stored data bytes across different types in TiFlash's PageStorage engine. Helps identify storage space usage patterns and detect potential storage imbalances or abnormal growth in specific data types.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_page_data_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 601, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot read", "used_for": "Used for monitoring the distribution of hot read regions across TiKV stores, showing both peer and leader replica counts. Essential for diagnosing read hotspot imbalances and verifying whether PD is properly distributing hot regions to prevent performance bottlenecks on specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_peer\"}", "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_leader\"}", "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",name=~\".*threshold\",type=~\"read.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 511, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for tracking the direction and frequency of hot region peer movements (move-peer and move-leader operations) across TiKV stores to balance write hotspots. Essential for diagnosing whether PD's hotspot scheduler is actively redistributing write-intensive regions and identifying stores that are frequently sources or destinations of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 123, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Number of Tables", "group": "PageStorage", "used_for": "Used for monitoring the number of tables stored in TiFlash across different storage pool modes (V2, V3, and mixed mode) per instance. Helps track table distribution and identify storage mode transitions or imbalances in TiFlash's columnar storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 273, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Eviction Duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time taken to evict regions from TiKV's in-memory engine cache. Critical for diagnosing memory pressure issues and understanding cache eviction performance when the in-memory engine needs to free up space for new data.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_eviction_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12056, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Subscriptions Data GC Lag", "group": "Event Store", "used_for": "**Used For:**\n\nUsed for monitoring the lag between subscription data and garbage collection in TiCDC's event store at p80, p95, and maximum percentiles. Critical for identifying data retention issues and ensuring timely cleanup of processed events, which prevents memory pressure and storage bloat in the event store.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [ 95 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.8, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 272, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Load Count", "group": "In Memory Engine", "used_for": "Used for monitoring the rate of region loading operations into TiKV's in-memory engine. Helps identify memory engine activity patterns and diagnose issues related to region data loading frequency or potential bottlenecks in the in-memory caching layer.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_in_memory_engine_load_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 602, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store read rate bytes", "group": "Statistics - hot read", "used_for": "**Used For:**\n\nUsed for monitoring read throughput (bytes per second) across TiKV stores to identify hot read regions and unbalanced read loads. Essential for diagnosing read hotspots, evaluating store-level read performance, and determining if read traffic is evenly distributed across the cluster.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20034, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Write Requests / s", "group": "Event Store", "used_for": "Used for monitoring the rate of write requests per second to TiCDC's event store across instances. Helps identify write throughput patterns and detect instances experiencing abnormal write loads or potential bottlenecks in event persistence.", "component": "TICDC", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_write_requests_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 274, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Eviction Count", "group": "In Memory Engine", "used_for": "Used for monitoring the number of region evictions from TiKV's in-memory engine. Helps diagnose memory pressure issues and identify when regions are being evicted from cache due to insufficient memory capacity.", "component": "TIKV", "phenomenon": "memory", "object": "region", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_in_memory_engine_eviction_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20002, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Input Bytes / s", "group": "Event Store", "used_for": "Used for monitoring the rate of data being written to TiCDC's event store in bytes per second. Helps identify data ingestion bottlenecks, unusual write traffic patterns, and capacity planning needs for the event store component.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_write_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 345, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PS Apply edits OPS By Instance", "group": "PageStorage", "used_for": "Used for monitoring the rate of PageStorage edit operations per TiFlash instance, broken down by operation type. Essential for diagnosing TiFlash write throughput issues and identifying instances with abnormal PageStorage modification patterns or bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_page_apply_edit_type{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 232, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PS Command OPS By Instance", "group": "PageStorage", "used_for": "Used for monitoring the rate of PageStorage commands (operations per second) across TiFlash instances, broken down by command type. Essential for identifying which TiFlash nodes are handling the most PageStorage operations and detecting imbalanced workload distribution or unusual command patterns that may indicate performance bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_page_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 603, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store read rate keys", "group": "Statistics - hot read", "used_for": "**Used For:**\n\nUsed for monitoring the read throughput in terms of keys per second across TiKV stores, comparing actual store-level metrics with expected hot region read rates. Essential for identifying read hotspots, diagnosing unbalanced read load distribution, and validating whether PD's hot region scheduling is effectively balancing read-intensive workloads across stores.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 275, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "In Memory Engine", "used_for": "Used for monitoring write operation latency in TiKV's In-Memory Engine. Critical for diagnosing performance issues when data is being written to the memory storage layer, helping identify bottlenecks in the in-memory write path.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20035, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Compressed Rows / s", "group": "Event Store", "used_for": "Used for monitoring the rate of row compression operations per second in TiCDC's event store. Helps assess event store compression efficiency and identify instances with abnormal compression activity that may indicate data processing bottlenecks or storage optimization issues.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_compressed_rows_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 605, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot read", "used_for": "Used for monitoring the direction and frequency of hot read region leader transfers across TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing read-heavy regions and identify stores that are frequently transferring leaders in or out to resolve read hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 604, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Store read query", "group": "Statistics - hot read", "used_for": "Used for monitoring read query rates across TiKV stores to identify hot read regions and unbalanced read traffic distribution. Essential for diagnosing read hotspots, evaluating load balancing effectiveness, and determining if specific stores are handling disproportionate read workloads that could impact cluster performance.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12044, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Input Event Count / s", "group": "Event Store", "used_for": "Used for monitoring the rate of events being ingested into TiCDC's event store per second, broken down by instance and event type. Essential for tracking data change capture throughput and identifying bottlenecks in the event processing pipeline.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_input_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 84, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Throughput", "group": "Rate Limiter", "used_for": "Used for monitoring the I/O throughput controlled by TiFlash's I/O limiter, broken down by operation type and instance. Essential for verifying that I/O rate limiting is working as configured and identifying instances where I/O throttling may be impacting query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22453, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Write Worker Busy Ratio", "group": "Event Store", "used_for": "Used for monitoring the I/O busy ratio of TiCDC event store write workers, indicating how much time workers spend on I/O operations versus idle time. Essential for identifying write bottlenecks and determining if write workers are saturated, which could impact CDC replication performance and latency.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_write_worker_io_duration_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance,db,worker) / sum(rate(ticdc_event_store_write_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[2m])) by (instance,db,worker) *100" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 305, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Threshold", "group": "Rate Limiter", "used_for": "Used for monitoring the current I/O rate limiting thresholds applied to TiFlash storage operations. Helps identify when I/O throttling is active and diagnose performance bottlenecks caused by rate limiting on read/write operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_io_limiter_curr{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (type, instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22225, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Write Duration", "group": "Event Store", "used_for": "Used for monitoring the latency of write operations to TiCDC's event store, showing maximum, p99, and average durations per instance. Critical for diagnosing TiCDC replication lag and identifying performance bottlenecks in the event storage layer that could impact change data capture throughput.", "component": "TICDC", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "sum(rate(ticdc_event_store_write_duration_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance) /\nsum(rate(ticdc_event_store_write_duration_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22360, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Dropped Resolve Lock Tasks / s", "group": "Log Puller", "used_for": "Used for monitoring the rate of dropped resolve lock tasks per second in TiCDC's log puller component. Helps identify when TiCDC is unable to process lock resolution tasks, which may indicate resource pressure, backlog issues, or potential data replication delays.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum(rate(ticdc_subscription_client_resolve_lock_task_drop_count[1m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 609, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Selector read events", "group": "Statistics - hot read", "used_for": "Used for monitoring the rate of hot read region detection events in PD's scheduler. Helps diagnose read hotspot identification efficiency and troubleshoot issues with PD's hot region balancing mechanism for read-heavy workloads.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 606, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot read", "used_for": "**Used For:**\n\nUsed for tracking the direction and frequency of hot region peer movements (move-peer and move-leader operations) across TiKV stores during read hotspot balancing. Essential for diagnosing whether PD's hotspot scheduler is actively rebalancing read-heavy regions and identifying which stores are sources (out) or destinations (in) of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 277, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Prepare for write duration", "group": "In Memory Engine", "used_for": "Used for monitoring the time spent preparing write operations in TiKV's in-memory engine. Critical for diagnosing write path bottlenecks and performance degradation in the memory storage layer before data is persisted to disk.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 607, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Hot cache read entry numbers", "group": "Statistics - hot read", "used_for": "Used for monitoring the total number of hot read regions cached in PD's hotspot detection system. Essential for understanding read hotspot distribution across stores and diagnosing whether PD is correctly identifying and tracking read-intensive regions for scheduling decisions.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 266, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Pending Rate and Duration", "group": "Rate Limiter", "used_for": "Used for monitoring the rate of I/O requests waiting in TiFlash's I/O limiter queue and their pending duration. Essential for identifying I/O throttling issues and diagnosing when storage operations are being rate-limited, which can impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_io_limiter_pending_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20032, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Write Batch Size", "group": "Event Store", "used_for": "Used for monitoring the distribution of write batch sizes in TiCDC's event store. Helps diagnose write efficiency and identify whether events are being batched optimally, which is critical for understanding I/O patterns and potential performance bottlenecks in change data capture operations.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_event_store_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 701, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Scheduler is running", "group": "Scheduler", "used_for": "Used for monitoring whether PD schedulers are currently allowed to run in the cluster. Essential for verifying scheduler availability and diagnosing issues when data rebalancing or region scheduling operations are not occurring as expected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"allow\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 276, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% In-memory engine write duration per server", "group": "In Memory Engine", "used_for": "Used for monitoring the 99th and 99.99th percentile write latency to TiKV's in-memory engine on each server instance. Critical for identifying write performance bottlenecks and tail latency issues in the in-memory storage layer, helping diagnose slow write operations that may impact transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 278, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% In-memory engine prepare for write duration per server", "group": "In Memory Engine", "used_for": "Used for monitoring the 99th and 99.99th percentile latency of write preparation operations in TiKV's in-memory engine, broken down by server instance. Critical for identifying performance bottlenecks and tail latency issues in the in-memory engine's write path, helping diagnose slow write operations that may impact transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22234, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Data Size On Disk", "group": "Event Store", "used_for": "Used for monitoring the total disk space consumed by TiCDC's event store on each instance. Critical for tracking storage usage trends, preventing disk exhaustion, and identifying instances that may need capacity expansion or data cleanup.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_store_on_disk_data_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 86, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Current Pending Count", "group": "Rate Limiter", "used_for": "Used for monitoring the number of write requests currently queued in TiFlash's I/O rate limiter, broken down by background and foreground writes. Essential for diagnosing write throughput bottlenecks and identifying when I/O limiting is causing request queuing, which can impact query performance and data ingestion latency.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20030, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Write Batch Event Count", "group": "Event Store", "used_for": "Used for monitoring the distribution of events per write batch in TiCDC's event store. Helps diagnose event batching efficiency and identify whether write operations are properly batched, which is critical for optimizing TiCDC replication throughput and reducing I/O overhead.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_event_store_write_batch_events_count_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 238, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Index & Filter Size", "group": "Kv Engine", "used_for": "Used for monitoring the memory consumption of RocksDB block cache components in TiKV, specifically tracking the size of index blocks and bloom filter blocks. Essential for diagnosing memory usage patterns and optimizing block cache configuration when experiencing memory pressure or cache efficiency issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n kv_engine_index_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "sum((\n kv_engine_filter_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 280, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "In Memory Engine", "used_for": "Used for monitoring the P95 latency of seek operations in TiKV's In-Memory Engine. Critical for diagnosing performance issues when queries scan or seek through data stored in memory, helping identify whether memory engine operations are causing query slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(1,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.95,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_in_memory_engine_seek_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_seek_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22235, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Data Size In Memory", "group": "Event Store", "used_for": "Used for monitoring the total size of event data currently held in memory by TiCDC's event store on each instance. Critical for detecting memory pressure issues and potential OOM risks when event processing lags behind data ingestion.", "component": "TICDC", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_store_in_memory_data_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 62, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Stall Duration", "group": "Storage Write Stall", "used_for": "Used for monitoring the 99th percentile duration of write stalls in TiFlash's storage engine. Critical for diagnosing write performance degradation and identifying when TiFlash experiences backpressure due to compaction, memory pressure, or I/O bottlenecks that temporarily block write operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, instance))", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 281, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Oldest Auto GC SafePoint", "group": "In Memory Engine", "used_for": "Used for monitoring the oldest automatically managed garbage collection safe point in TiKV's in-memory engine. Helps identify potential memory pressure issues when the safe point lags significantly behind current time, indicating delayed cleanup of obsolete data versions.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20004, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Scan Requests / s", "group": "Event Store", "used_for": "Used for monitoring the rate of scan requests per second in TiCDC's event store across instances. Helps identify event scanning activity patterns and diagnose potential bottlenecks when TiCDC is reading change events from the event store for replication.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_scan_requests_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 704, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Balance leader scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's balance leader scheduler, which redistributes region leaders across TiKV stores to balance load. Essential for diagnosing leader distribution imbalances and verifying that the scheduler is actively working to maintain cluster balance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 703, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Balance Region movement", "group": "Scheduler", "used_for": "Used for monitoring the rate of region movements in and out of each TiKV store during balance operations. Essential for diagnosing load balancing behavior, identifying stores that are shedding or receiving excessive regions, and troubleshooting uneven data distribution issues in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\", direction=\"out\"}[2m])) by (store)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\", direction=\"in\"}[2m])) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 286, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Compaction Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the duration of remote compaction requests in TiKV's cloud worker service. Essential for diagnosing performance issues with offloaded compaction operations and identifying bottlenecks in cloud-native storage optimization workflows.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(increase(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Throughput", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage write throughput and delta management operations. Helps diagnose write stall issues by comparing data ingestion rates (write/ingest) against background delta merge and compaction activities, essential for identifying storage bottlenecks during heavy write workloads.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[2m]))", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"write|ingest\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 90, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring the rate of write commands executed by each TiFlash instance, broken down by command type. Essential for identifying write throughput bottlenecks, detecting write stalls, and diagnosing which specific TiFlash instances are experiencing storage write performance issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance, type)", "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 705, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Balance Region scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's balance-region scheduler, which automatically redistributes regions across TiKV nodes to maintain cluster balance. Essential for diagnosing uneven data distribution issues and verifying that region rebalancing is occurring as expected during cluster scaling or hotspot mitigation.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22300, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Scan Operation Duration ", "group": "Event Store", "used_for": "Used for monitoring the 99th percentile latency of event store read operations in TiCDC's new architecture. Critical for diagnosing performance bottlenecks in change data capture when TiCDC reads events from its internal storage, helping identify slow scan operations that could delay replication.", "component": "TICDC", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 702, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Balance leader movement", "group": "Scheduler", "used_for": "Used for monitoring the balance of Raft leader movements across TiKV stores during leader scheduling. Helps diagnose uneven leader distribution and identify stores that are excessively sending or receiving leaders, which can indicate load balancing issues or hotspot problems in the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\", direction=\"out\"}[2m])) by (store)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\", direction=\"in\"}[2m])) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 279, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Iterator operations", "group": "In Memory Engine", "used_for": "Used for monitoring iterator operation rates in TiKV's In-Memory Engine, including seek, seek_found, and next operations. Essential for diagnosing memory engine scan performance and understanding how efficiently the in-memory storage layer handles data traversal operations.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Newest Auto GC SafePoint", "group": "In Memory Engine", "used_for": "Used for monitoring the most recent garbage collection safe point in TiKV's in-memory engine. Helps track memory cleanup progress and diagnose issues where stale data isn't being properly garbage collected, which could lead to memory pressure.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 283, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Auto GC SafePoint Gap", "group": "In Memory Engine", "used_for": "Used for monitoring the gap between the newest and oldest safe points in TiKV's in-memory engine, measured in units of 2^18. Helps identify potential memory pressure issues and GC efficiency problems when the gap grows too large, indicating that old data versions are not being cleaned up promptly.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20006, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Scan Bytes / s", "group": "Event Store", "used_for": "Used for monitoring the rate of bytes scanned per second by TiCDC's event store, broken down by instance and scan type. Essential for tracking data throughput during change data capture operations and identifying potential bottlenecks in event processing or replication lag issues.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_scan_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 284, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Auto GC SafePoint Gap With TiKV", "group": "In Memory Engine", "used_for": "Used for monitoring the gap between TiDB's GC safe point and TiKV's in-memory engine safe point, measured in units of 2^18 (approximately 262K). Critical for detecting memory pressure issues when the in-memory engine falls behind GC operations, which could lead to memory bloat or data retention problems.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 167, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Stale Read OPS", "group": "Raft", "used_for": "Used for monitoring the rate of stale read operations per second on each TiFlash instance. Helps track the usage of stale read feature and identify instances handling high volumes of stale read requests, which can indicate read-heavy workloads tolerating slightly outdated data for better performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_stale_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 35, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Read Index OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations per second in TiFlash instances. Helps identify read throughput patterns and diagnose issues with stale read requests or Raft consensus overhead when TiFlash queries data from TiKV.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 137, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Total", "group": "Storage Write Stall", "used_for": "Used for monitoring the total I/O throughput of TiFlash storage operations, separated into write/ingest operations versus delta management operations (compaction, GC, etc.). Essential for diagnosing write stalls and identifying whether performance bottlenecks are caused by incoming writes or background maintenance tasks competing for I/O resources.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"})", "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!~\"write|ingest\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 270, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Learner Read Failures", "group": "Raft", "used_for": "Used for monitoring the rate of learner read failures in TiFlash's Raft replication, categorized by failure type. Critical for diagnosing data consistency issues and identifying problems with TiFlash's ability to read data from TiKV learner replicas.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_learner_read_failures_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 710, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Rule checker", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's rule checker, which validates and enforces placement rules for data distribution across the cluster. Essential for diagnosing issues with replica placement policies, label constraints, and identifying when rule violations are being detected or corrected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"rule_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 288, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Remote Processing Requests", "group": "Cloud Worker Service", "used_for": "Used for monitoring the number of remote processing requests handled by TiKV cloud workers, including remote compaction and coprocessor requests. Essential for tracking workload distribution in disaggregated TiKV deployments and diagnosing performance issues related to remote compute operations.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum((\n tikv_worker_remote_compact_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) ", "sum((\n tikv_worker_remote_cop_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 706, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Balance Hot Region scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's hot region balance scheduler, which automatically redistributes regions with high I/O load across TiKV nodes. Essential for diagnosing uneven I/O distribution issues and verifying whether the scheduler is actively balancing hot spots in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20024, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Subscription Num", "group": "Event Store", "used_for": "Used for monitoring the number of active event subscriptions in TiCDC's event store. Helps identify subscription management issues, potential memory pressure from excessive subscriptions, and diagnose replication pipeline bottlenecks when subscription counts are abnormally high or low.", "component": "TICDC", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_event_store_subscription_num{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12060, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Resolved Ts Lag", "group": "Schema Store", "used_for": "Used for monitoring the lag between the schema store's resolved timestamp and current time in TiCDC. Critical for identifying schema synchronization delays that could impact change data capture performance and data consistency in downstream systems.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_schema_store_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12062, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Register Table Num", "group": "Schema Store", "used_for": "Used for monitoring the number of tables currently registered in TiCDC's schema store. Helps track schema synchronization scope and diagnose issues when tables aren't being captured or replicated as expected.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_schema_store_register_table_num{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 709, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Patrol Region time", "group": "Scheduler", "used_for": "Used for monitoring the maximum time taken by PD to patrol (scan) all regions in the cluster. Critical for diagnosing PD performance issues when region metadata synchronization is slow or when the cluster has scaling problems due to large numbers of regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(max(pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})by(instance))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12066, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Get Table Info Duration", "group": "Schema Store", "used_for": "Used for monitoring the latency distribution of table metadata retrieval operations in TiCDC's schema store. Critical for diagnosing slow schema lookups that can impact replication performance and identifying bottlenecks in metadata access patterns.", "component": "TICDC", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_schema_store_get_table_info_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 711, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Replica checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of replica checker events in PD's scheduler. Helps diagnose replica placement issues, such as missing replicas, incorrect replica counts, or replica distribution imbalances that trigger the checker to take corrective actions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"replica_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20026, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Shared Column Schema Count", "group": "Schema Store", "used_for": "Used for monitoring the number of shared column schemas cached in TiCDC's schema store. Helps diagnose memory usage and schema synchronization efficiency, particularly when investigating schema-related performance issues or memory pressure in TiCDC instances.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_common_shared_column_schema_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 271, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Index Events", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations in TiFlash, categorized by event type. Essential for diagnosing read consistency issues and understanding the frequency of linearizable read requests that require Raft consensus verification.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_read_index_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 713, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Filter source", "group": "Scheduler", "used_for": "Used for monitoring the rate at which PD's scheduler filters out stores from different sources during scheduling operations. Helps diagnose why certain stores are being excluded from scheduling decisions and identify potential scheduling bottlenecks or misconfigurations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_source_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (source, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12064, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Get Table Info Count / s", "group": "Schema Store", "used_for": "Used for monitoring the rate of table information retrieval operations per second in TiCDC's schema store. Helps identify schema metadata access patterns and diagnose potential bottlenecks when TiCDC frequently queries table definitions during data replication.", "component": "TICDC", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_schema_store_get_table_info_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 712, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region merge checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of region merge operations triggered by PD's merge checker. Essential for diagnosing region fragmentation issues and verifying that small adjacent regions are being properly merged to optimize cluster performance and reduce metadata overhead.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"merge_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 290, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Concurrency Limiter Waiting Length", "group": "Cloud Worker Service", "used_for": "Used for monitoring the number of requests waiting in TiKV's concurrency limiter queue. Helps identify when worker threads are saturated and requests are being throttled, indicating potential resource contention or the need to adjust concurrency limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum((\n tikv_worker_limiter_waiting_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 714, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Filter target", "group": "Scheduler", "used_for": "Used for monitoring the rate at which PD's scheduler filters out potential target stores during scheduling operations. Helps diagnose why certain stores are being excluded from scheduling decisions and identify potential scheduling bottlenecks or misconfigurations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_target_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (target, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 37, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Wait Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash waiting for Raft log index synchronization at p80, p95, and p99 percentiles. Critical for diagnosing data replication delays between TiKV and TiFlash, especially when queries show stale data or TiFlash read performance degrades due to slow log application.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, $additional_groupby) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, $additional_groupby))", "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, $additional_groupby))", "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, $additional_groupby))", "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 287, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Compaction Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the latency of remote compaction requests in TiKV's cloud worker service, tracking p99 and p9999 percentiles. Critical for diagnosing performance issues with disaggregated storage compaction operations and identifying when remote compaction workers are experiencing delays or bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22305, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Wait Resolved Ts Duration", "group": "Schema Store", "used_for": "Used for monitoring the latency (p80, p95, p100) of waiting for resolved timestamps in TiCDC's schema store component. Critical for diagnosing replication delays caused by schema synchronization bottlenecks, especially when changefeeds experience lag due to slow schema resolution.", "component": "TICDC", "phenomenon": "latency", "object": "store", "percentile": [ 80, 95 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(ticdc_schema_store_wait_resolved_ts_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_schema_store_wait_resolved_ts_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(1.0, sum(rate(ticdc_schema_store_wait_resolved_ts_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 292, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the latency distribution of snapshot requests in TiKV's cloud worker service. Critical for diagnosing performance issues with remote coprocessor snapshot operations, which can impact distributed query execution and data consistency checks.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(increase(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 82, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration", "group": "Raft", "used_for": "Used for monitoring the latency of applying Raft write logs in TiFlash, showing p99, p100 (max), and average durations. Critical for diagnosing write replication delays from TiKV to TiFlash and identifying performance bottlenecks in the Raft log application process that could impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[2m]))", "sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[2m])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[2m]) ) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (decode)", "group": "Raft", "used_for": "Used for monitoring the time spent decoding Raft write data in TiFlash regions. Critical for diagnosing TiFlash write path performance issues, particularly when data ingestion from TiKV is slow or experiencing bottlenecks during the decode phase of region replication.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12034, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scan Duration", "group": "Event Service", "used_for": "Used for monitoring the distribution of scan operation latencies in TiCDC's Event Service component. Essential for diagnosing performance bottlenecks in change data capture event scanning and identifying when event processing is experiencing delays.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scan_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 801, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "scatter operator event", "group": "Scatter and Splitter", "used_for": "Used for monitoring the rate of scatter operator events in PD's scheduler, tracking successful, failed, and skipped scatter operations per minute. Essential for diagnosing region distribution issues, load balancing problems, and understanding why scatter operations may be failing or being skipped during cluster rebalancing.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"skip\"}[2m])*60) by (event)", "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fail\"}[2m])*60", "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"success\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 93, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft admin logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft admin commands in TiFlash through a heatmap visualization. Essential for diagnosing performance issues related to administrative operations like region splits, merges, and configuration changes in TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (write blocks)", "group": "Raft", "used_for": "Used for monitoring the latency of writing Raft log data blocks to TiFlash storage during region replication. Critical for diagnosing TiFlash write performance bottlenecks and identifying issues with Raft data persistence that could impact data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 295, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Copr Request Process duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the processing latency of coprocessor requests in TiKV's cloud worker service, tracking both p99 and p9999 percentiles. Essential for diagnosing slow coprocessor query performance and identifying tail latency issues in distributed query execution across worker instances.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 291, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Concurrency Limiter Waiting Duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring how long requests wait when hitting TiKV's concurrency limiter across worker services. Critical for identifying when worker thread pools are saturated and requests are being throttled, which directly impacts query latency and throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster", "worker_instance" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22472, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scan Duration", "group": "Event Service", "used_for": "Used for monitoring the p999 latency of TiCDC's event service scan operations, which read change events from TiKV. Critical for diagnosing replication lag issues and identifying performance bottlenecks in the event capture pipeline when TiCDC experiences slow data synchronization.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scan_duration_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance) /\nsum(rate(ticdc_event_service_scan_duration_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "histogram_quantile(0.999, sum(rate(ticdc_event_service_scan_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 293, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the latency of snapshot requests in TiKV's remote coprocessor worker service, tracking p99 and p9999 percentiles. Critical for diagnosing performance issues in distributed query execution when TiDB pushes computation to TiKV storage nodes.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 294, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Copr Request Process duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the processing duration of coprocessor requests handled by TiKV's cloud worker service. Essential for diagnosing performance bottlenecks in distributed query execution, particularly when queries involve table scans or aggregations pushed down to TiKV storage nodes.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(increase(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 802, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "scatter store selection", "group": "Scatter and Splitter", "used_for": "Used for monitoring the distribution of scattered regions across TiKV stores, showing both leader and follower replica placement rates. Helps diagnose load balancing issues during region scatter operations, such as table creation or manual scatter commands, by identifying if certain stores are receiving disproportionate numbers of scattered regions.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"false\"}[2m])*60) by (store)", "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"true\"}[2m])*60) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 36, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Batch Read Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of batch read index operations in TiFlash's Raft layer at p80, p95, and p99 percentiles. Critical for diagnosing read performance issues and identifying when TiFlash experiences delays in obtaining consistent read snapshots from the Raft consensus protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, $additional_groupby) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, $additional_groupby))", "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22262, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scanned Entry Count", "group": "Event Service", "used_for": "Used for monitoring the rate of entries scanned by TiCDC's Event Service per instance. Helps identify data processing throughput and detect instances with abnormally high or low scanning activity, which may indicate replication bottlenecks or uneven workload distribution.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scanned_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 81, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the distribution of time spent applying Raft write log commands in TiFlash nodes. Essential for diagnosing replication lag and write performance bottlenecks when TiFlash is synchronizing data from TiKV through the Raft protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 901, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "gRPC Completed commands rate", "group": "gRPC", "used_for": "Used for monitoring the rate of completed gRPC commands handled by each PD instance, broken down by gRPC method type. Essential for tracking PD's API throughput, identifying which operations are most frequently called, and detecting abnormal spikes or drops in command processing that may indicate cluster coordination issues.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 903, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "gRPC commands concurrency number", "group": "gRPC", "used_for": "Used for monitoring the maximum concurrent gRPC API requests being processed by PD servers. Essential for identifying potential bottlenecks when concurrent request limits are reached, which can cause client timeouts and cluster coordination issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"grpc\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 289, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Remote Requests Failures", "group": "Cloud Worker Service", "used_for": "Used for monitoring the failure rate of remote worker requests in TiKV's cloud worker service, including remote compaction and coprocessor operations. Critical for diagnosing distributed task execution issues and identifying when remote workers are experiencing errors that could impact query performance or storage operations.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(rate(\n tikv_worker_remote_compact_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_worker_remote_cop_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22264, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scanned Entry Bytes / s", "group": "Event Service", "used_for": "Used for monitoring the rate of bytes scanned and skipped per second by TiCDC's Event Service during change data capture operations. Helps identify inefficient scanning patterns where high skipped bytes relative to scanned bytes may indicate suboptimal event filtering or unnecessary data processing overhead.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_store_scan_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"scanned\"}[2m])) by (instance)", "sum(rate(ticdc_event_store_scan_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"skipped\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 241, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Frequent Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of frequent Raft events per second in TiFlash, broken down by event type. Helps identify abnormal Raft activity patterns, replication issues, or excessive leader transfers that may impact TiFlash's data synchronization performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_raft_frequent_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 235, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft consensus events occurring in TiFlash nodes, broken down by event type. Essential for diagnosing Raft replication issues, understanding cluster coordination activity, and identifying abnormal spikes in Raft protocol operations that may indicate network problems or region leadership changes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_raft_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20015, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Resolved Ts Lag", "group": "Event Service", "used_for": "Used for monitoring the lag between the current time and resolved timestamp in TiCDC's event service, broken down by instance and type. Critical for identifying replication delays and ensuring timely data synchronization from TiDB to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_service_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22263, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scanned Transaction Count", "group": "Event Service", "used_for": "Used for monitoring the rate of transactions scanned by TiCDC's Event Service per instance. Essential for tracking CDC workload intensity and identifying instances processing unusually high transaction volumes, which may indicate replication bottlenecks or uneven load distribution.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scanned_txn_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 902, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "gRPC 99% Completed commands duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC command completion in PD servers. Critical for identifying slow gRPC operations and diagnosing communication bottlenecks between PD and other TiDB components (TiKV, TiDB servers).", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, grpc_method, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 296, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Copr Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the latency distribution of coprocessor requests processed by TiKV's cloud worker service. Essential for diagnosing performance issues in distributed query execution, particularly when queries involve table scans or aggregations pushed down to TiKV storage nodes.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(increase(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 904, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "gRPC Received commands rate", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC commands received by each PD instance, broken down by gRPC method type. Essential for identifying communication bottlenecks, unusual traffic patterns, or specific command types causing high load on the PD cluster.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_msg_received_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 905, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "gRPC Error rate", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC requests in PD, broken down by method type. Essential for identifying communication issues between PD and other cluster components (TiDB/TiKV), helping diagnose cluster coordination problems and service availability issues.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", grpc_type=\"unary\", grpc_code!=\"OK\"}[2m])) by (grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 297, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Copr Request duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the latency of remote coprocessor requests processed by TiKV cloud workers, tracking p99 and p9999 percentiles. Essential for diagnosing performance issues in distributed query execution, particularly when TiDB pushes computation down to TiKV storage nodes, helping identify slow coprocessor operations that impact query response times.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 907, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Failed Region Request", "group": "gRPC", "used_for": "Used for monitoring the rate of failed region-related requests to PD, broken down by request type, caller ID, and caller component. Essential for diagnosing communication issues between PD and other TiDB components (TiKV, TiDB) when region metadata operations fail.", "component": "PD", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_region_request_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", event=\"failed\"}[2m])) by (request, caller_id, caller_component)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20018, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Finished Scan Task Count", "group": "Event Service", "used_for": "Used for monitoring the rate of completed scan tasks in TiCDC's Event Service per instance. Helps identify scan task throughput and detect instances with abnormally low task completion rates, which may indicate performance bottlenecks or processing issues in the change data capture pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scan_task_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 20025, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Pending Scan Task", "group": "Event Service", "used_for": "Used for monitoring the rate of pending scan tasks in TiCDC's Event Service, which handles change data capture operations. Helps identify bottlenecks in event processing when scan tasks accumulate faster than they can be processed, indicating potential replication lag or resource constraints.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_pending_scan_task_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 240, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft eager garbage collection operations in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential performance impacts from excessive cleanup operations in the Raft layer.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_eager_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22265, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Scanned Transaction Bytes / s", "group": "Event Service", "used_for": "Used for monitoring the rate of transaction data bytes scanned by TiCDC's Event Service per second. Helps identify data processing throughput and diagnose potential bottlenecks in change data capture when replication lag occurs or when investigating I/O performance issues in the event service layer.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scanned_dml_size_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 301, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR RPC Duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the RPC request latency of native backup and restore operations in TiKV's cloud worker service. Critical for diagnosing performance issues during backup/restore tasks and identifying slow BR operations that may impact cluster availability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22266, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Available Memory", "group": "Event Service", "used_for": "Used for monitoring the available memory quota in TiCDC's Event Service component. Critical for detecting memory pressure and potential OOM issues that could impact change data capture performance and stability.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_service_available_memory_quota{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 244, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Region Size (by event) Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of region data sizes during flush events in TiFlash, showing both flushed and unflushed region bytes as a heatmap. Helps identify region size patterns and diagnose I/O performance issues related to uneven data distribution or abnormally large regions during Raft log application.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"unflushed\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"flushed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 300, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory Limiter Usage", "group": "Cloud Worker Service", "used_for": "Used for monitoring current memory consumption by TiKV's worker memory limiter across instances. Helps identify memory pressure issues and potential out-of-memory risks in TiKV worker threads, essential for capacity planning and preventing memory-related performance degradation.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum((\n tikv_worker_memory_limiter_current_used\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 234, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Log Gap Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log gaps between applied and compact indexes in TiFlash instances. Helps identify Raft log replication lag issues and potential memory pressure from accumulated uncommitted logs, which can impact TiFlash's ability to serve fresh data.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"applied_index\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"compact_index\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1003, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "HTTP commands concurrency number", "group": "HTTP", "used_for": "Used for monitoring the maximum concurrent HTTP API requests being processed by PD servers. Helps identify potential API bottlenecks, overload conditions, or abnormal spikes in concurrent requests that could impact PD's responsiveness and cluster management operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"http\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 906, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Success Region Request(1% sample)", "group": "gRPC", "used_for": "Used for monitoring the rate of successful region-related requests to PD's gRPC API, broken down by request type and caller information (with 1% sampling). Essential for tracking PD's region metadata operation throughput and identifying which components are actively querying region information.", "component": "PD", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_region_request_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", event=\"success\"}[2m])) by (request, caller_id, caller_component)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 299, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Remote Copr Request Operations", "group": "Cloud Worker Service", "used_for": "**Used For:**\n\nUsed for monitoring the rate of remote coprocessor requests processed by TiKV cloud workers, including DAG queries, analyze operations, and checksum calculations. Essential for tracking workload distribution in disaggregated TiKV deployments and identifying bottlenecks in remote compute operations.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(rate(\n tikv_worker_remote_cop_dag_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_worker_remote_analyze_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_worker_remote_checksum_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1002, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "HTTP 99% Completed commands duration", "group": "HTTP", "used_for": "Used for monitoring the 99th percentile latency of HTTP API commands handled by PD servers. Critical for identifying slow API responses and diagnosing performance issues in PD's HTTP service layer that may impact cluster management operations.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, service, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 302, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR RPC Operations", "group": "Cloud Worker Service", "used_for": "Used for monitoring the rate of native Backup & Restore RPC operations in TiKV's cloud worker service. Essential for tracking backup/restore activity and diagnosing issues with cloud-native BR operations by operation type.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_worker_native_br_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 303, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR Backup Operations", "group": "Cloud Worker Service", "used_for": "Used for monitoring the success and error rates of native backup and restore (BR) operations in TiKV's cloud worker service. Essential for tracking backup job health, identifying backup failures, and ensuring data protection operations are completing successfully across cluster instances.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n native_br_backup_success\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "sum(rate(\n native_br_backup_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22258, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Dispatcher Status", "group": "Event Service", "used_for": "Used for monitoring the count and status distribution of event dispatchers in TiCDC's Event Service. Essential for tracking dispatcher health, identifying stuck or failed dispatchers, and diagnosing data replication issues in the new TiCDC architecture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_event_service_dispatcher_status_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22259, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Scanned Entry Count / s", "group": "Event Service", "used_for": "Used for monitoring the rate of entries scanned per second by TiCDC's Event Service across instances. Helps identify data scanning throughput and detect potential bottlenecks in change data capture processing when replication lag occurs.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_scanned_count_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 264, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Big Write To Region Size Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of large write operations to TiFlash regions over time as a heatmap. Helps identify write hotspots and diagnose uneven data distribution issues when certain regions receive disproportionately large write volumes, which can impact TiFlash performance and storage balance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"big_write_to_region\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 298, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Remote Copr Response Size", "group": "Cloud Worker Service", "used_for": "Used for monitoring the rate of response data size returned from remote coprocessor requests in TiKV's cloud worker service, including DAG queries, analyze operations, and checksum tasks. Essential for identifying network bandwidth consumption and diagnosing issues with data transfer efficiency between TiDB and remote TiKV workers in disaggregated storage architectures.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster", "worker_instance" ], "promql": [ "sum(rate(\n tikv_worker_remote_cop_dag_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_worker_remote_analyze_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tikv_worker_remote_checksum_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1001, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "HTTP Completed commands rate", "group": "HTTP", "used_for": "Used for monitoring the rate of completed HTTP commands handled by PD instances, broken down by service type. Helps identify HTTP API usage patterns, detect traffic anomalies, and diagnose potential bottlenecks in PD's HTTP service layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, service)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 261, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Committed Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of committed write sizes in TiFlash's Raft replication layer as a heatmap. Helps identify write size patterns and detect anomalies in data replication workloads, which is critical for diagnosing TiFlash I/O performance issues and understanding data ingestion characteristics.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write_committed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1102, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Handle transactions rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd transaction operations handled by PD instances, broken down by success/failure results. Essential for diagnosing PD's etcd transaction throughput and identifying potential bottlenecks or failures in metadata operations that could impact cluster coordination.", "component": "PD", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22260, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Service Channel Size", "group": "Event Service", "used_for": "Used for monitoring the size of internal event channels in TiCDC's Event Service component. Helps identify event processing bottlenecks and backpressure issues when channels accumulate too many pending events, which can indicate slow downstream consumption or processing delays.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_channel_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22469, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Reset Dispatcher / s", "group": "Event Service", "used_for": "Used for monitoring the rate of dispatcher resets per second in TiCDC's event service. Helps identify instability issues in change data capture pipelines, as frequent dispatcher resets may indicate connection problems, resource constraints, or abnormal event processing failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_service_reset_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 243, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Entry Batch Size Heatmap", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log entry batch sizes in TiFlash nodes through a heatmap visualization. Helps identify abnormal replication patterns, batch size variations, and potential bottlenecks in the Raft log synchronization process between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_entry_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"normal\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 305, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR Restored Snapshot Bytes", "group": "Cloud Worker Service", "used_for": "Used for monitoring the rate of snapshot data restored by TiKV's native Backup & Restore (BR) feature, measured in bytes per second per instance. Essential for tracking restore operation progress and throughput during cluster recovery or data migration scenarios.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n native_br_restored_snapshot_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1101, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "MVCC DB total size", "group": "etcd", "used_for": "Used for monitoring the total size and actual usage of PD's embedded etcd database, which stores cluster metadata and scheduling information. Essential for detecting metadata storage growth issues, potential disk space problems, and identifying when etcd database maintenance or compaction is needed.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}", "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10054, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sent Message Count Per Second", "group": "Message Center", "used_for": "Used for monitoring TiCDC's message center throughput, tracking the rate of sent, dropped, and error messages per second between components. Essential for diagnosing message delivery issues, identifying communication bottlenecks, and detecting abnormal message drop rates in TiCDC's new architecture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_messaging_error_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, target, type)", "sum(rate(ticdc_messaging_drop_message_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, target, type)", "sum(rate(ticdc_messaging_send_message_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, target, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 304, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR Restore Errors", "group": "Cloud Worker Service", "used_for": "Used for monitoring error rates during native Backup & Restore (BR) operations in TiKV, including general restore errors by type and specific WAL epoch overwrite errors by instance. Critical for diagnosing BR restore failures and identifying problematic TiKV nodes during disaster recovery or data migration operations.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n native_br_restore_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n native_br_restore_rfengine_wal_epoch_overwritten_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 239, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC Duration", "group": "Raft", "used_for": "Used for monitoring the latency of Raft eager garbage collection operations in TiFlash at p95 and p99 percentiles. Critical for diagnosing performance issues related to TiFlash's Raft log cleanup process, which can impact replication efficiency and storage management when GC operations become slow.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_eager_gc_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type) / sum(rate(tiflash_raft_eager_gc_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 260, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Raft throughput", "group": "Raft", "used_for": "Used for monitoring the rate of Raft log replication throughput in TiFlash instances, broken down by operation type. Essential for diagnosing data synchronization performance between TiKV and TiFlash, and identifying bottlenecks in the Raft replication pipeline.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 71, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Keys flow", "group": "Raft", "used_for": "Used for monitoring the rate of key operations flowing through TiFlash's Raft layer, categorized by operation type. Helps identify data replication patterns and diagnose issues with data synchronization between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 251, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Ingest Level Distribution", "group": "Kv Engine", "used_for": "Used for monitoring the distribution of SST file ingestion across different LSM-tree levels in TiKV's RocksDB engine. Helps diagnose write amplification issues and understand data compaction patterns, particularly useful when investigating storage performance degradation or unusual disk I/O behavior.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n kv_engine_ingest_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 306, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Native BR Restore Snapshot Request Duration", "group": "Cloud Worker Service", "used_for": "Used for monitoring the duration of native Backup & Restore (BR) snapshot restore requests in TiKV's cloud worker service. Essential for diagnosing restore performance issues and tracking the time taken to complete snapshot restoration operations in cloud-native deployments.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n native_br_restore_snapshot_request_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10050, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Dispatcher Manager Count", "group": "Dispatcher", "used_for": "Used for monitoring the number of active table dispatcher managers in TiCDC's new architecture. Helps identify dispatcher resource allocation issues and track how many tables are being actively replicated across changefeeds.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanagermanager_event_dispatcher_manager_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1103, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% Handle transactions duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in PD. Critical for diagnosing slow consensus operations and identifying performance degradation in PD's metadata storage layer that could impact cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22447, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Slow Message Count", "group": "Message Center", "used_for": "Used for monitoring the count of slow message handling operations in TiCDC's message center, categorized by instance and message type. Essential for identifying message processing bottlenecks and diagnosing replication lag issues caused by slow message handling in TiCDC's new architecture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_messaging_slow_handle_counter{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 310, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC Files", "group": "Local File GC", "used_for": "Used for monitoring the rate of local file garbage collection operations in TiKV's Raft store. Helps identify file cleanup activity and potential issues with storage space management or excessive temporary file accumulation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n rfstore_local_file_gc_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1104, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% Peer round trip time seconds", "group": "etcd", "used_for": "Used for monitoring the 99th percentile network round-trip time between etcd peers in the PD cluster. Critical for diagnosing etcd communication latency issues that can impact PD's consensus performance and cluster metadata synchronization.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, To, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 308, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "On Disk Files", "group": "Local File GC", "used_for": "Used for monitoring the number of local files currently stored on disk in TiKV's Raft store. Helps identify potential disk space issues and track the effectiveness of local file garbage collection processes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n rfstore_local_file_on_disk\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 91, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency", "group": "Raft", "used_for": "Used for monitoring the latency of Raft log replication from TiKV to TiFlash at p95 and p99 percentiles. Critical for diagnosing data synchronization delays between TiKV and TiFlash, helping identify network issues or upstream bottlenecks affecting real-time analytics performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le))", "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 296, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Log Replication Rejected", "group": "Raft", "used_for": "Used for monitoring the rate of rejected Raft log append operations in TiFlash nodes. Helps identify replication bottlenecks or resource constraints that prevent TiFlash from accepting log entries from TiKV, which can lead to data synchronization delays.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1107, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% backend commit duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd backend commit operations in PD. Critical for diagnosing etcd performance issues and disk I/O bottlenecks that can impact cluster metadata operations and leader election stability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1106, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "etcd disk wal fsync rate", "group": "etcd", "used_for": "Used for monitoring the rate of WAL (Write-Ahead Log) fsync operations per minute in PD's etcd storage. Helps identify write throughput patterns and diagnose potential disk I/O bottlenecks when etcd experiences high write loads or performance degradation.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 309, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Alive Files", "group": "Local File GC", "used_for": "Used for monitoring the number of active local files in TiKV's Raft store that are managed by the garbage collection system. Helps identify potential file accumulation issues or GC inefficiency that could lead to disk space problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n rfstore_local_file_alive\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20167, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Dispatcher Count", "group": "Dispatcher", "used_for": "Used for monitoring the number of active table event dispatchers in TiCDC's new architecture. Helps identify dispatcher distribution issues, resource allocation problems, and potential bottlenecks when replicating changes from multiple tables across the cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanager_table_event_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (instance, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1108, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "etcd disk backend commit rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd backend commit operations per minute in PD. Essential for tracking etcd write throughput and identifying potential bottlenecks in metadata persistence that could impact cluster coordination performance.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 20037, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Trigger Dispatcher Count", "group": "Dispatcher", "used_for": "Used for monitoring the number of table trigger event dispatchers in TiCDC's new architecture. Helps track dispatcher resource allocation and identify potential bottlenecks when replicating table-level change events across changefeeds.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dispatchermanager_table_trigger_event_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (instance, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 92, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log replication latency from TiKV to TiFlash instances. Essential for diagnosing data synchronization delays and identifying network or replication bottlenecks between TiKV and TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 312, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Pending GC Files", "group": "Local File GC", "used_for": "Used for monitoring the number of local files pending garbage collection on each TiKV instance. Helps identify potential disk space issues when old files accumulate due to GC delays or failures, which could lead to storage exhaustion.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n rfstore_local_file_pending_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10056, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Create Dispatcher Duration", "group": "Dispatcher", "used_for": "Used for monitoring the latency of creating dispatchers in TiCDC's new architecture at p90, p95, and p99 percentiles. Critical for diagnosing slow dispatcher initialization issues that could delay change data capture processing and impact replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(ticdc_sink_create_dispatcher_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "histogram_quantile(0.95, sum(rate(ticdc_sink_create_dispatcher_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "histogram_quantile(0.99, sum(rate(ticdc_sink_create_dispatcher_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 311, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Skip GC Files", "group": "Local File GC", "used_for": "**Used For:**\n\nUsed for monitoring the number of local files skipped during garbage collection in TiKV's Raft store. Helps identify potential issues with file cleanup processes and diagnose storage space management problems when files are being retained longer than expected.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n rfstore_local_file_skip_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1105, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% WAL fsync duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile fsync latency of etcd's Write-Ahead Log (WAL) in PD. Critical for diagnosing PD performance issues caused by slow disk I/O, as high WAL fsync duration directly impacts cluster metadata operations and can lead to leader election delays or cluster instability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 290, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Prehandle Throughput Heatmap", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the distribution of Raft snapshot prehandling throughput across TiFlash instances as a heatmap. Helps identify performance bottlenecks and uneven workload distribution during snapshot processing, which is critical for diagnosing slow region replication and data synchronization issues in TiFlash.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_command_throughput_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"prehandle_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1607, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% Endpoint health check latency", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd endpoint health checks in PD. Critical for detecting etcd communication issues and cluster coordination problems that could impact PD's ability to manage cluster metadata and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_etcd_endpoint_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}[2m])) by (instance, endpoint, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1115, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Endpoint health state", "group": "etcd", "used_for": "Used for monitoring the health status of etcd client endpoints in PD. Critical for detecting etcd connectivity issues and ensuring PD can properly communicate with its embedded etcd cluster for metadata storage and coordination.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_client{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 242, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Heavy Raft Apply Duration", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the 99th percentile latency of heavy Raft operations (snapshots and SST ingestion) in TiFlash. Critical for diagnosing slow data replication issues and identifying performance bottlenecks when TiFlash applies Raft commands from TiKV, especially during region transfers or bulk data ingestion.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 250, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Size Heatmap", "group": "Raft Snapshot / IngestSST", "used_for": "Used for visualizing the distribution of Raft snapshot sizes in TiFlash over time as a heatmap. Helps identify abnormal snapshot size patterns, detect large snapshot transfers that may impact network bandwidth, and diagnose replication performance issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_snapshot_total_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"approx_raft_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22223, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Handle Message Duration", "group": "Dispatcher", "used_for": "Used for monitoring the latency distribution of TiCDC's event collector when processing change data capture events. Critical for diagnosing performance bottlenecks in the event collection pipeline and identifying delays in replication processing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_event_collector_handle_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 20014, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Received Resolved Ts / s", "group": "Dispatcher", "used_for": "Used for monitoring the rate of resolved timestamp events received by TiCDC dispatchers per second. Helps diagnose data replication lag and event processing bottlenecks by tracking how frequently watermark progress updates are being received across TiCDC instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dispatcher_received_event_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"ResolvedTs\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1110, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Raft committed index", "group": "etcd", "used_for": "Used for monitoring the committed index of the Raft consensus protocol in PD's embedded etcd cluster. Essential for tracking etcd replication progress and diagnosing consensus issues, data synchronization delays, or split-brain scenarios in the PD cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"committedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 262, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Uncommitted Size Heatmap", "group": "Raft Snapshot / IngestSST", "used_for": "Used for visualizing the distribution of uncommitted Raft snapshot sizes in TiFlash over time as a heatmap. Helps identify snapshot size patterns and detect abnormally large uncommitted snapshots that could indicate replication bottlenecks or memory pressure issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"snapshot_uncommitted\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 316, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flow controller actions (Store)", "group": "Flow Control", "used_for": "Used for monitoring the rate of flow control throttling actions at the TiKV store level. Helps diagnose when and how frequently TiKV is applying backpressure to regulate write traffic due to I/O pressure or resource constraints.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",level=\"store\"}\n [$__rate_interval]\n)) by (level, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 249, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Ongoing raft snapshot", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the rate of ongoing Raft snapshot data transfer in TiFlash, broken down by snapshot type. Essential for diagnosing replica synchronization issues, identifying snapshot bottlenecks, and understanding data replication overhead during region migrations or node recoveries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10052, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Dispatcher Request Handle Result", "group": "Dispatcher", "used_for": "Used for monitoring the success and failure rates of dispatcher request handling in TiCDC's new architecture. Essential for diagnosing data replication issues and identifying problems in the dispatcher's ability to process change events from upstream TiDB to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_handle_dispatcher_request{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12040, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Registered Dispatcher Count", "group": "Dispatcher", "used_for": "Used for monitoring the number of registered dispatchers in TiCDC's event collector service per instance. Helps diagnose event processing capacity issues and identify instances with abnormal dispatcher counts that may indicate resource constraints or replication bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_event_service_dispatcher_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 72, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode Duration", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the time spent pre-decoding Raft snapshots in TiFlash before they are applied to the storage engine. Critical for diagnosing slow snapshot transfer and recovery issues, especially when TiFlash nodes are catching up with data or recovering from failures.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"snapshot_predecode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 73, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Flush Duration", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the latency distribution of Raft snapshot flush operations in TiFlash. Critical for diagnosing performance issues during region snapshot application, which can impact data replication and recovery times when TiFlash nodes receive snapshots from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"snapshot_flush\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 89, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Write Throughput By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring write and ingest throughput (bytes per second) for each TiFlash instance. Essential for identifying write performance bottlenecks, detecting storage write stalls, and diagnosing uneven write distribution across TiFlash nodes that could impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write\"}[2m])) by (instance)", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"ingest\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 315, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flow controller actions (Region)", "group": "Flow Control", "used_for": "Used for monitoring the rate of flow control throttling actions triggered at the region level in TiKV. Helps diagnose when and how frequently TiKV applies backpressure to writes due to region-level resource constraints, indicating potential hotspot regions or write amplification issues.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n kv_engine_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",level=\"region\"}\n [$__rate_interval]\n)) by (level, type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1111, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Raft applied index", "group": "etcd", "used_for": "Used for monitoring the applied index of etcd's Raft log in PD, which indicates the last log entry that has been applied to the state machine. Essential for tracking etcd replication progress and diagnosing PD cluster consistency issues or replication lag between PD nodes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"appliedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1109, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Raft term", "group": "etcd", "used_for": "Used for monitoring the Raft term number in PD's embedded etcd cluster. Helps detect leader elections and cluster instability—frequent term changes indicate leadership churn, network partitions, or consensus issues that can impact PD's availability and cluster coordination.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"term\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 317, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler flow", "group": "Flow Control", "used_for": "Used for monitoring write flow and throttled flow in TiKV's scheduler per instance. Essential for identifying when flow control is actively throttling writes due to resource pressure, helping diagnose write performance degradation and backpressure issues in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22224, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Event Collector Receive Event Lag", "group": "Dispatcher", "used_for": "Used for monitoring the lag duration between when change events are generated and when they are received by TiCDC's event collector in the dispatcher component. Critical for identifying replication delays and diagnosing data synchronization latency issues in TiCDC's new architecture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dispatcher_received_event_lag_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1201, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "PD server TSO handle duration", "group": "TiDB", "used_for": "Used for monitoring the latency of PD server handling TSO (Timestamp Oracle) requests at p90, p99, and p999 percentiles. Critical for diagnosing transaction performance issues, as TSO allocation delays directly impact TiDB's ability to assign timestamps for distributed transactions and can cause cluster-wide latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 610, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Output Row Batch Count", "group": "Sink - General", "used_for": "Used for monitoring the number of rows batched together in each output operation from TiCDC sink to downstream systems. Critical for diagnosing sink throughput issues and optimizing batch size configurations, especially when investigating replication lag or downstream write performance problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=~\"$k8s_cluster\",sharedpool_id=~\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=~\"$k8s_cluster\",sharedpool_id=~\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / sum(rate(ticdc_sink_batch_row_count_count{k8s_cluster=~\"$k8s_cluster\",sharedpool_id=~\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 265, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Ingest Uncommitted Size Heatmap", "group": "Raft Snapshot / IngestSST", "used_for": "Used for visualizing the distribution of uncommitted SST file sizes during TiFlash's ingest operations. Helps identify abnormal ingestion patterns and diagnose issues with Raft snapshot application or SST file ingestion that could impact write performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"ingest_uncommitted\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 319, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Throttle duration", "group": "Flow Control", "used_for": "Used for monitoring the cumulative time TiKV scheduler spends in throttled state due to flow control mechanisms. Critical for diagnosing write performance degradation caused by flow control throttling, helping identify when the system is actively limiting request rates to prevent overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Applying snapshots Count", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the number of Raft snapshots currently being applied in TiFlash, including pending snapshots, prehandling subtasks, and parallel prehandling tasks. Essential for diagnosing snapshot application bottlenecks during region replication, data recovery, or when TiFlash nodes are catching up with data changes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumPrehandlingSubTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumParallelPrehandlingTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumWaitedParallelPrehandlingTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1202, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "PD client TSO handle duration", "group": "TiDB", "used_for": "Used for monitoring the latency of TSO (Timestamp Oracle) requests from TiDB to PD, showing average, p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD performance bottlenecks, as TSO allocation is required for every transaction in TiDB.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"tso.*\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"tso.*\"}[2m])) by (type)", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"tso.*\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"tso.*\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"tso.*\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 313, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC Errors", "group": "Local File GC", "used_for": "Used for monitoring the error rate of local file garbage collection operations in TiKV's Raft store. Critical for identifying file system issues, storage cleanup failures, or disk-related problems that could lead to space exhaustion or data management issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n rfstore_local_file_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (error, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20028, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "DS Input Channel Length", "group": "Dynamic Stream", "used_for": "Used for monitoring the input channel queue length in TiCDC's Dynamic Stream module. Helps identify event processing bottlenecks and backpressure issues when the channel accumulates too many pending events, indicating the downstream cannot keep up with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dynamic_stream_event_chan_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, module)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 320, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler throttled CF", "group": "Flow Control", "used_for": "Used for monitoring which column families (CFs) in TiKV are being throttled by the scheduler due to flow control mechanisms. Helps identify when write pressure causes TiKV to actively limit operations on specific column families to prevent resource exhaustion and maintain cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 322, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Non-txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time non-transactional commands (like raw KV operations) spend being throttled due to flow control mechanisms in TiKV. Critical for diagnosing performance degradation when non-txn workloads are being rate-limited to protect cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Ingest SST Duration", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the latency distribution of SST (Sorted String Table) file ingestion operations in TiFlash during Raft snapshot application. Critical for diagnosing slow data replication from TiKV to TiFlash and identifying performance bottlenecks in the snapshot restore process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"ingest_sst\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20036, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "DS Pending Queue Length", "group": "Dynamic Stream", "used_for": "Used for monitoring the length of pending queues in TiCDC's Dynamic Stream module. Helps identify data replication bottlenecks and backpressure issues when the queue accumulates, indicating that downstream processing cannot keep up with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_dynamic_stream_pending_queue_len{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",}[2m])) by (instance, module)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1203, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Handle TSO requests count", "group": "TiDB", "used_for": "Used for monitoring the rate of TSO (Timestamp Oracle) requests handled by PD servers. Essential for diagnosing timestamp allocation bottlenecks and understanding transaction workload patterns, as every distributed transaction in TiDB requires TSO allocation.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tso_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 321, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time transaction commands spend being throttled due to flow control mechanisms in TiKV. Critical for diagnosing performance degradation caused by resource pressure, helping identify when transactions are being delayed by backpressure from storage layer congestion or memory limits.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 612, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Output Row Count (per second)", "group": "Sink - General", "used_for": "Used for monitoring the rate of rows being written to downstream sinks per second in TiCDC, broken down by keyspace and changefeed. Essential for tracking replication throughput and identifying performance bottlenecks or stalls in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 615, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Error Count / m", "group": "Sink - General", "used_for": "Used for monitoring the rate of sink execution errors per minute in TiCDC changefeeds. Critical for identifying data replication failures and downstream write issues that could cause data loss or replication lag.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1626, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "PD server QueryRegion handle duration", "group": "TiDB", "used_for": "**Used For:**\n\nUsed for monitoring the latency of TiDB querying region information from PD server at p90, p99, and p999 percentiles. Critical for diagnosing slow query issues caused by region metadata lookup delays, especially when TiDB experiences high tail latency in accessing PD's region routing information.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(pd_server_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_server_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_server_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_core_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_core_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.999, sum(rate(pd_core_query_region_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 327, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler running tasks", "group": "Scheduler", "used_for": "Used for monitoring the number of currently executing tasks in TiKV's scheduler thread pool per instance. Helps identify scheduler bottlenecks, thread pool saturation, and uneven task distribution across TiKV nodes that could impact transaction processing performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1204, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "TSO request batch size", "group": "TiDB", "used_for": "Used for monitoring the batch size of TSO (Timestamp Oracle) requests from TiDB to PD, showing p90, p99, and average values. Helps diagnose timestamp allocation efficiency and identify whether TiDB is effectively batching TSO requests, which is critical for transaction performance optimization.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [ 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "sum(rate(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(rate(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 613, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Output DDL Executing Duration", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile execution duration of DDL operations at TiCDC's sink output stage. Critical for diagnosing downstream database performance issues and identifying slow DDL replication that could cause changefeed lag or replication delays.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_ddl_exec_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_ddl_exec_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 324, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the total number of scheduler operations and \"too busy\" rejections in TiKV's transaction scheduler. Essential for diagnosing write contention issues, scheduler overload conditions, and identifying when TiKV is rejecting requests due to excessive load or resource constraints.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22261, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Output DDL Count / Minutes", "group": "Sink - General", "used_for": "Used for monitoring the rate of DDL (Data Definition Language) operations executed by TiCDC sink per minute. Helps track DDL replication throughput and identify potential bottlenecks or delays in schema change propagation from source to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_sink_ddl_exec_duration_bucket{le=\"+Inf\",k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 127, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode SST to DT Duration", "group": "Raft Snapshot / IngestSST", "used_for": "Used for monitoring the time spent pre-decoding snapshot SST files into TiFlash's DeltaTree format during Raft snapshot operations. Critical for diagnosing slow snapshot ingestion and identifying bottlenecks in TiFlash's data replication process from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"snapshot_predecode_sst2dt\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 325, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiKV. Helps identify workload distribution across different priority queues and diagnose issues where high-priority operations may be blocked by lower-priority commands.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 618, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Conflict Detect Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction conflict detection in TiCDC's sink module. Critical for diagnosing replication delays caused by transaction conflicts when TiCDC writes data to downstream databases, helping identify bottlenecks in change data capture pipelines.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1628, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Handle QueryRegion requests count", "group": "TiDB", "used_for": "**Used For:**\n\nUsed for monitoring the rate of QueryRegion requests handled by PD, broken down by query type. Essential for diagnosing region lookup performance issues and identifying unusual patterns in region metadata queries from TiDB nodes.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_query_region_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(pd_core_query_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 326, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending commands in TiKV's scheduler queue per instance. Essential for identifying scheduler bottlenecks, command processing delays, and potential write stalls that could impact transaction performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_context_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 614, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Sink Running DDL Count / m", "group": "Sink - General", "used_for": "Used for monitoring the number of DDL operations currently being executed by TiCDC sinks per minute. Helps identify DDL execution bottlenecks and track schema change propagation progress in real-time replication scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_ddl_exec_running{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 318, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler discard ratio", "group": "Flow Control", "used_for": "Used for monitoring the ratio of discarded scheduler commands in TiKV due to flow control mechanisms. Helps identify when TiKV is rejecting requests because of resource pressure or I/O bottlenecks, indicating potential overload situations that require capacity adjustment or optimization.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 99, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the effectiveness of TiFlash's rough set filter in reducing data scanning during queries. Helps evaluate query optimization efficiency by showing the percentage of data blocks filtered out before full scanning, with higher rates indicating better filter performance and reduced I/O overhead.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (instance)", "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]))) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1301, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heartbeat region event QPS", "group": "Heartbeat", "used_for": "Used for monitoring the rate of region heartbeat events received by PD from TiKV stores. Essential for diagnosing region scheduling issues, detecting abnormal region state changes, and identifying stores with excessive region operations that may impact cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 97, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate Histogram", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the distribution of rough set filter effectiveness in TiFlash's columnar storage. Helps identify how efficiently TiFlash is pruning data blocks during query execution, with higher filter rates indicating better query performance through reduced data scanning.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 196, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload keys speed by type (all)", "group": "Disaggregated-Write", "used_for": "Used for monitoring the rate of checkpoint key uploads by type in TiFlash disaggregated storage architecture. Helps diagnose checkpoint upload performance and identify which key types are contributing most to I/O activity during data persistence operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_keys_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 328, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the total bytes being written by TiKV's scheduler component per instance. Helps identify write bottlenecks, uneven write distribution across TiKV nodes, and potential memory pressure issues when the scheduler's write buffer accumulates excessively.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 197, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow by type (incremental+compaction)", "group": "Disaggregated-Write", "used_for": "Used for monitoring the rate of checkpoint data upload flow in TiFlash disaggregated storage, broken down by operation type (incremental updates and compaction). Essential for diagnosing write throughput issues and understanding the balance between incremental checkpoint updates and background compaction activities in the write node.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 329, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler memory quota", "group": "Scheduler", "used_for": "Used for monitoring the memory quota usage of TiKV's scheduler component, comparing in-use memory against total capacity. Essential for diagnosing memory pressure issues in the scheduler and preventing out-of-memory errors that could impact transaction processing performance.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_use\"}\n \n)) by (instance) ", "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1629, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "QueryRegion request batch size", "group": "TiDB", "used_for": "Used for monitoring the batch size of QueryRegion requests from TiDB to PD, showing p90, p99, and average values. Helps diagnose region query efficiency and identify whether TiDB is batching region lookups effectively, which impacts query routing performance and PD load.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [ 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_client_request_handle_query_region_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_query_region_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "sum(rate(pd_client_request_handle_query_region_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type) / sum(rate(pd_client_request_handle_query_region_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1610, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region heartbeat handle latency overview", "group": "Heartbeat", "used_for": "**Used For:**\n\nUsed for monitoring the latency distribution (p99, p90, p80) of PD processing region heartbeat messages from TiKV stores. Critical for diagnosing PD performance bottlenecks when region heartbeat handling becomes slow, which can impact cluster scheduling decisions and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "histogram_quantile(0.9, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))", "sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) / sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 619, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the utilization rate of TiCDC transaction sink workers that flush data to downstream databases. Helps identify worker saturation issues and determine if additional worker capacity is needed to handle the replication workload efficiently.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_batch_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance,id) / sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance,id) *100" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 335, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Scheduled Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the average time tasks wait in the scheduler's Tokio thread pool queue before execution in TiKV. Critical for identifying thread pool saturation and scheduling delays that can impact overall transaction processing performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1302, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% Region heartbeat handle latency by store", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling region heartbeats from each TiKV store. Critical for diagnosing PD performance bottlenecks and identifying stores with slow heartbeat processing that could impact cluster metadata synchronization and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 333, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Running Tasks", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the number of actively running tasks in TiKV's scheduler worker pool. Helps identify scheduler saturation and potential bottlenecks when tasks accumulate, indicating insufficient worker threads or slow task processing that could impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 191, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Duration", "group": "Disaggregated-Write", "used_for": "Used for monitoring the 99th percentile latency of remote object lock requests in disaggregated TiFlash write operations. Critical for diagnosing performance bottlenecks in distributed lock acquisition when TiFlash compute and storage nodes are separated.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 330, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler Pool Wait Duration", "group": "Scheduler", "used_for": "Used for monitoring the wait time before scheduler tasks are executed in TiKV's thread pools. Critical for diagnosing scheduler bottlenecks and thread pool saturation that can cause transaction processing delays and increased write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 634, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Input Rows / s", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the rate of rows being processed by TiCDC's transaction sink workers per second. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when rows aren't being consumed at expected rates.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance,id)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 176, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote File Num", "group": "Disaggregated-Write", "used_for": "Used for monitoring the number of remote storage files in TiFlash's disaggregated storage architecture. Essential for tracking storage distribution and identifying potential issues with remote file accumulation or cleanup in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"num_files\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 653, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC workers sending messages to message queue (MQ) sinks during data replication. Critical for identifying tail latency issues in the MQ sink pipeline and diagnosing slow message delivery that could impact replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1627, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "PD client QueryRegion handle duration", "group": "TiDB", "used_for": "Used for monitoring the latency of TiDB's PD client when querying region information from PD, showing average, p90, p99, and p999 percentiles. Critical for diagnosing slow query issues caused by region metadata lookup delays or PD communication bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 90, 99, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"query_region.*\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"query_region.*\"}[2m])) by (type)", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"query_region.*\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"query_region.*\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"query_region.*\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 332, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Schedule Wait Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the time tasks spend waiting in TiKV's scheduler worker pool queue before execution. Critical for diagnosing scheduler bottlenecks and thread pool saturation that can cause transaction processing delays and increased query latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1304, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region heartbeat report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV nodes, as missing or delayed heartbeats indicate network problems or store unavailability that could affect cluster metadata accuracy and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"ok\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1303, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "TiKV side heartbeat statistics", "group": "Heartbeat", "used_for": "Used for monitoring heartbeat communication between TiKV nodes and PD (Placement Driver), tracking both heartbeat message rates by type and pending heartbeat counts. Essential for diagnosing cluster coordination issues, detecting network problems between TiKV and PD, and identifying when TiKV nodes fail to report their status timely, which can lead to incorrect scheduling decisions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1305, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region schedule push", "group": "Heartbeat", "used_for": "Used for monitoring the rate at which PD pushes region scheduling commands to TiKV stores via heartbeats. Essential for diagnosing scheduling delays and identifying stores that may be slow to receive or process region migration commands.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1609, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Concurrent Runner Pending Task", "group": "Heartbeat", "used_for": "Used for monitoring the number of pending tasks in PD's rate-limited concurrent runner queue. Helps identify heartbeat processing bottlenecks when tasks accumulate, indicating PD is overwhelmed or experiencing performance degradation in handling cluster heartbeat requests.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_ratelimit_runner_pending_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 175, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Store Usage", "group": "Disaggregated-Write", "used_for": "Used for monitoring remote storage usage in disaggregated TiFlash architecture, tracking total size, valid data size, and storage efficiency ratio across instances. Essential for diagnosing remote storage capacity issues, identifying storage bloat, and ensuring efficient data management in TiFlash's write node separation model.", "component": "TIFLASH", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"total_size\"}) by (instance)", "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"valid_size\"}) by (instance)", "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"total_size\"})) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 628, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Outgoing Bytes", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the outgoing byte rate from TiCDC to Kafka when using the MQ sink. Essential for tracking data replication throughput, identifying network bottlenecks, and diagnosing slow data delivery to downstream Kafka clusters.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 630, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Request Latency", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the latency of Kafka producer requests in TiCDC's message queue sink. Essential for diagnosing downstream Kafka performance issues and identifying bottlenecks in change data replication to Kafka topics.", "component": "TICDC", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 620, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Full Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of full transaction flush operations in TiCDC's sink workers. Critical for diagnosing downstream write performance bottlenecks and identifying when transaction batches take unusually long to commit to the target system.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 334, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Poll Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the average time TiKV scheduler worker threads spend polling and executing tasks. Critical for diagnosing scheduler performance bottlenecks and identifying when worker threads are overloaded or experiencing excessive task processing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "((\n tikv_scheduler_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 189, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Request QPS", "group": "Disaggregated-Write", "used_for": "Used for monitoring the rate of remote object lock requests per second in TiFlash's disaggregated storage architecture. Essential for diagnosing lock contention and coordination overhead in write-heavy workloads when TiFlash compute and storage are separated.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_disaggregated_object_lock_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Status", "group": "Disaggregated-Write", "used_for": "Used for monitoring the status of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for tracking S3 object cleanup activities and diagnosing issues with storage space reclamation in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_storage_s3_gc_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance,type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1608, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Concurrent Runner Failed Task", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed tasks in PD's concurrent runner, which handles heartbeat processing and scheduling operations. Essential for identifying PD internal task execution issues that could lead to increased heartbeat latency or cluster coordination problems.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_ratelimit_runner_failed_tasks_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1335, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heartbeat Performance Duration BreakDown (Accumulation)", "group": "Heartbeat", "used_for": "Used for analyzing the time breakdown of PD's region heartbeat processing pipeline, showing accumulated duration across different processing stages and lock wait times. Essential for diagnosing PD heartbeat latency bottlenecks by identifying which specific stage (handling, lock acquisition) is causing delays in processing TiKV region heartbeats.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_core_region_heartbeat_breakdown_handle_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (name)", "sum(rate(pd_core_acquire_regions_lock_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 635, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Backend Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of DML batch commits to the downstream database in TiCDC's transaction sink. Critical for diagnosing slow transaction replication issues and identifying backend database performance bottlenecks that may cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 193, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Duration Breakdown", "group": "Disaggregated-Write", "used_for": "Used for monitoring the p99 and p999 latency breakdown of remote garbage collection operations in TiFlash's disaggregated storage (S3). Critical for diagnosing performance bottlenecks in cloud-native TiFlash deployments where GC operations on remote storage may impact write node efficiency and overall cluster performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1630, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Concurrent Runner Pending Duration", "group": "Heartbeat", "used_for": "Used for monitoring the maximum waiting time of tasks in PD's rate limiter concurrent runner queue. Critical for identifying bottlenecks in heartbeat processing when tasks are queuing excessively, which can indicate PD is overloaded or experiencing performance degradation in handling cluster coordination tasks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_ratelimit_runner_task_max_waiting_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 337, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "First Poll Count", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the rate of first-time task polls in TiKV's scheduler worker thread pool. Helps diagnose task scheduling efficiency and identify whether the worker pool is processing new tasks promptly or experiencing delays in task pickup.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1306, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region heartbeat report error", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed region heartbeat reports from TiKV stores to PD. Critical for identifying communication issues between TiKV and PD that could lead to stale region metadata, scheduling delays, or cluster instability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"err\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 627, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Inflight Requests", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of in-flight (pending) requests to Kafka brokers from TiCDC's message queue sink. Critical for identifying backpressure issues, network bottlenecks, or Kafka broker slowness that could cause replication lag or CDC pipeline stalls.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 631, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Records Per Request", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of Kafka records batched per produce request in TiCDC's MQ sink. Helps evaluate batching efficiency and diagnose throughput issues—higher values indicate better batching, while low values may suggest configuration problems or performance bottlenecks in the Kafka producer.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 187, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload Duration", "group": "Disaggregated-Write", "used_for": "Used for monitoring the tail latency (p99 and p999) of checkpoint upload operations in TiFlash's disaggregated storage architecture. Critical for diagnosing slow checkpoint persistence to remote storage, which can impact write performance and data durability in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type, $additional_groupby) / 1000000000)", "histogram_quantile(0.999, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 251, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FAP result", "group": "Disaggregated-Write", "used_for": "Used for monitoring the result distribution of FAP (Fast Add Peer) tasks in TiFlash disaggregated storage architecture. Helps diagnose write node data synchronization issues by tracking success/failure rates of peer addition operations, which are critical for maintaining data consistency across TiFlash compute and storage nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_fap_task_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 340, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of TiKV scheduler commands at p99 and p9999 percentiles. Critical for diagnosing transaction processing delays and identifying performance bottlenecks in TiKV's transaction scheduler, especially when investigating slow writes or lock contention issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1307, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Region heartbeat report active", "group": "Heartbeat", "used_for": "Used for monitoring the rate of active region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster coordination problems that could impact region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"bind\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 629, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Request Rate", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of requests sent from TiCDC to Kafka brokers per second. Essential for tracking message publishing throughput and identifying bottlenecks in the CDC replication pipeline to Kafka.", "component": "TICDC", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 174, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow", "group": "Disaggregated-Write", "used_for": "Used for monitoring the data upload throughput to remote storage in disaggregated TiFlash architecture, tracking both incremental checkpoint uploads and compaction-related data flows. Essential for diagnosing write path performance issues, identifying upload bottlenecks, and ensuring efficient data persistence in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"incremental\"}[2m])) by ($additional_groupby)", "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"compaction\"}[2m])) by ($additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 336, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Idle Duration", "group": "Scheduler Worker Pool", "used_for": "Used for monitoring the average idle time of worker threads in TiKV's scheduler thread pool. Helps identify thread pool efficiency issues—high idle duration may indicate underutilization, while consistently low values could signal thread pool saturation or insufficient worker threads for handling scheduler tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_scheduler_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1308, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% store heartbeat handle duration", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling store heartbeat requests from TiKV nodes. Critical for diagnosing PD performance issues when stores report slow heartbeat processing, which can indicate PD overload or scheduling bottlenecks affecting cluster health detection.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1310, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heartbeat stream report error", "group": "Heartbeat", "used_for": "Used for monitoring the rate of failed region heartbeat messages sent from PD to TiKV stores. Critical for diagnosing communication issues between PD and TiKV that can cause cluster instability, stale region information, or scheduling delays.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"err\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 252, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FAP state", "group": "Disaggregated-Write", "used_for": "Used for monitoring the state distribution of FAP (Fast Add Peer) tasks in TiFlash's disaggregated storage architecture. Helps diagnose write node operations and peer replication issues by tracking task state transitions and identifying bottlenecks in the disaggregated write path.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_fap_task_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 709, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Encoder Group Input Channel Size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the input channel buffer size of TiCDC's encoder group in MQ sink scenarios. Helps identify backpressure issues when the channel fills up, indicating that encoders cannot keep pace with incoming change events, which may lead to replication lag.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 339, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the total rate of scheduler operations and \"too busy\" rejections for specific command types in TiKV. Essential for diagnosing scheduler bottlenecks, identifying when TiKV is overloaded and rejecting requests, and understanding the throughput of different transaction commands.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 710, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Encoder Group Output Channel Size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the output channel buffer size of TiCDC's encoder group when replicating data to message queue sinks. Helps identify backpressure issues and bottlenecks in the encoding pipeline when the channel fills up, indicating that downstream MQ consumption is slower than upstream encoding.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 254, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FAP time by stage", "group": "Disaggregated-Write", "used_for": "Used for monitoring the 99.9th percentile latency of TiFlash FAP (Fast Add Peer) operations across different stages in disaggregated storage architecture. Critical for diagnosing write path performance bottlenecks and identifying slow peer replication stages in TiFlash's distributed write process.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, sum(round(1000000000*rate(tiflash_fap_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type, $additional_groupby) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 253, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "FAP no match reason", "group": "Disaggregated-Write", "used_for": "Used for tracking reasons why TiFlash's Fast Add Peer (FAP) optimization cannot be applied during data replication in disaggregated storage mode. Helps diagnose write path inefficiencies by identifying why the fast path is bypassed, enabling optimization of data synchronization between write and compute nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_fap_nomatch_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 186, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Flow", "group": "Disaggregated-Compute", "used_for": "Used for monitoring the rate of data flow through TiFlash's remote cache in disaggregated storage-compute architecture. Helps identify cache hit/miss patterns and diagnose performance issues related to remote data access between compute and storage nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_remote_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 342, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiKV, tracking both average and tail percentiles (p99, p9999). Essential for identifying commands that scan excessive keys, which can indicate inefficient queries, large range scans, or potential hotspot issues affecting read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 185, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Operations", "group": "Disaggregated-Compute", "used_for": "Used for monitoring remote cache performance in TiFlash disaggregated storage architecture, tracking cache operations rates and hit ratios for both DTFile and page-level caches. Essential for diagnosing I/O efficiency issues and optimizing cache effectiveness when compute and storage nodes are separated.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit|dtfile_miss\"}[2m]))", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit|page_miss\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 626, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Kafka Producer Compression Ratio", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the compression efficiency of Kafka messages produced by TiCDC's MQ sink. Helps identify compression performance issues and optimize network bandwidth usage when replicating data to Kafka, with lower ratios indicating better compression.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 188, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Usage", "group": "Disaggregated-Compute", "used_for": "Used for monitoring remote cache capacity and utilization in TiFlash's disaggregated storage architecture. Essential for tracking DTFile cache and page cache usage to optimize cache hit rates and diagnose performance issues in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_DTFileCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_DTFileCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1309, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heartbeat stream report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful heartbeat region messages pushed from PD to TiKV stores. Essential for diagnosing cluster communication health and identifying stores with heartbeat delivery issues that could indicate network problems or store unavailability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"ok\"}[2m])*60) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 345, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations specifically on the lock column family during scheduler command execution in TiKV. Helps diagnose lock-related performance issues and identify excessive lock scanning that may indicate transaction conflicts or inefficient query patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 173, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Read Duration Breakdown", "group": "Disaggregated-Compute", "used_for": "Used for monitoring the 99th percentile breakdown of read operation latency in TiFlash's disaggregated compute layer. Essential for identifying performance bottlenecks in distributed read operations and diagnosing which stage (network, storage access, or computation) is causing slow queries in the disaggregated architecture.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 711, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Batch Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of batch processing in TiCDC's MQ (Message Queue) sink worker. Critical for identifying tail latency issues when TiCDC replicates data changes to downstream message queue systems like Kafka, helping diagnose slow batch operations that could impact replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 344, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's scheduler for specific command types. Helps diagnose scan performance issues, identify excessive scanning behavior, and understand the workload characteristics of read-heavy operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 343, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per transaction command in TiKV's scheduler at different percentiles (p99, p9999) and average. Essential for identifying write-heavy transactions, detecting abnormal batch sizes, and diagnosing performance issues caused by large-scale write operations that may impact scheduler throughput.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 712, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Batch Size Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile batch size of messages sent by TiCDC's MQ sink workers. Critical for diagnosing message batching efficiency and identifying whether the sink is properly aggregating changes before sending to message queues, which directly impacts throughput and downstream processing performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 346, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations in TiKV's write column family during scheduler command execution. Helps diagnose write-heavy workload patterns and identify potential performance bottlenecks caused by excessive scanning in write operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1502, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "History last index", "group": "Region storage", "used_for": "**Used For:**\n\nUsed for monitoring the last Raft log index synchronized by PD's region syncer. Essential for diagnosing region metadata synchronization lag and detecting potential issues with PD's region information consistency across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"last_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1311, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "99% Bucket Report Interval", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile interval between bucket heartbeat reports from TiKV stores to PD. Critical for diagnosing heartbeat delays and detecting stores that are slow to report region bucket information, which can impact PD's scheduling decisions and cluster metadata accuracy.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 741, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Claim Check Send Message Count", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of claim check messages sent by TiCDC's MQ sink when handling large messages. Helps diagnose message offloading behavior and identify when TiCDC is using external storage for oversized change events that exceed the message queue's size limits.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 7111, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Send CheckpointTs Message Count", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of checkpoint timestamp messages sent by MQ sink workers in TiCDC's new architecture. Helps diagnose checkpoint propagation issues and ensures proper coordination between sink workers when replicating data to message queue systems like Kafka.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(ticdc_sink_mq_checkpoint_ts_message_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 236, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "MVCCIndexCache", "group": "Disaggregated-Compute", "used_for": "Used for monitoring the MVCC index cache hit rate and access patterns in TiFlash disaggregated compute nodes. Essential for diagnosing metadata lookup performance issues and determining if cache sizing is adequate for the workload, particularly when experiencing slow query performance in the compute layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, instance)", "sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"hit\"}[2m])) by (instance) /sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1501, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Syncer index", "group": "Region storage", "used_for": "Used for monitoring the synchronization index of PD's region syncer, which tracks the progress of region metadata synchronization across PD nodes. Essential for diagnosing region information consistency issues and detecting synchronization lag that could impact cluster metadata accuracy.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"sync_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1312, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Bucket Report State", "group": "Heartbeat", "used_for": "Used for monitoring the rate and status of bucket reports sent from TiKV stores to PD during heartbeat communication. Essential for diagnosing bucket-level load balancing issues and detecting stores that fail to report bucket statistics, which can impact region scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, type,status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 178, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 Bytes", "group": "S3", "used_for": "Used for monitoring S3 storage I/O throughput in TiFlash, tracking read/write bytes per second including DMFile operations. Essential for diagnosing S3 storage performance bottlenecks, identifying abnormal I/O patterns, and understanding TiFlash's cloud storage usage when using disaggregated storage architecture.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteDMFileBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 349, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Check memory locks duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time spent checking memory locks in TiKV's transaction scheduler. Critical for diagnosing transaction processing delays and identifying contention issues when multiple transactions attempt to access the same data in memory.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 348, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler command read duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of read operations processed by TiKV's scheduler, tracking both P99 and P9999 percentiles. Essential for diagnosing read performance bottlenecks and identifying when the scheduler is experiencing delays in processing read commands, which directly impacts query response times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 347, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations in TiKV's scheduler for the default column family. Helps diagnose scan-heavy workload patterns and identify potential performance issues related to range queries or table scans in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 353, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PlaceIndexTask/Reuse OPS", "group": "Disaggregated-Compute", "used_for": "Used for monitoring the operations per second of index placement and reuse tasks in TiFlash disaggregated compute nodes. Helps diagnose index management efficiency and identify potential bottlenecks in distributed index operations across compute instances.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_place_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type, $additional_groupby)", "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"place_index_update\"}[$__rate_interval])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1602, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "State of all instances", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync state across all PD instances in the cluster. Essential for verifying DR replication status and quickly identifying instances that may have fallen out of sync or entered abnormal states during disaster recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 356, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PlaceIndex Tasks Duration", "group": "Disaggregated-Compute", "used_for": "**Used For:**\n\nUsed for monitoring the P99 and P9999 latency of PlaceIndex tasks in TiFlash's disaggregated compute layer. Critical for diagnosing performance bottlenecks in TiFlash's index placement operations, especially when experiencing slow query execution or storage layer delays in disaggregated architecture deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"place_index_update\"}[$__rate_interval]))) by (le,type, $additional_groupby) / 1000000000)", "histogram_quantile(0.9999, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"place_index_update\"}[$__rate_interval])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"place_index_update\"}[$__rate_interval])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1605, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Recovered region", "group": "DR Autosync", "used_for": "**Used For:**\n\nUsed for monitoring the number of regions that have been recovered in PD's disaster recovery (DR) auto-sync mechanism. Critical for tracking DR recovery progress after failures and verifying that data replication has been successfully restored across availability zones.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recovered_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 740, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Claim Check Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of claim check message sending operations in TiCDC's MQ sink. Critical for diagnosing performance bottlenecks when TiCDC uses claim check pattern to handle large messages in message queue-based replication, helping identify slow message delivery issues that could impact replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "rate(ticdc_sink_mq_claim_check_send_message_duration_sum[30s]) / rate(ticdc_sink_mq_claim_check_send_message_duration_count[30s])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1603, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Tick", "group": "DR Autosync", "used_for": "Used for monitoring the tick rate of PD's disaster recovery (DR) auto-sync mechanism. Helps verify that the DR synchronization heartbeat is functioning properly and can indicate issues with the DR replication coordination process.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 358, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "PlaceIndex update rows/deletes", "group": "Disaggregated-Compute", "used_for": "**Used For:**\n\nUsed for monitoring the rate and distribution of PlaceIndex update operations (row updates and deletes) in TiFlash's disaggregated compute layer, showing p99 and maximum latencies. Essential for diagnosing performance issues with TiFlash's metadata index maintenance and identifying bottlenecks in write-heavy workloads affecting compute nodes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_place_index_stats_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, $additional_groupby) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_place_index_stats_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))", "sum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))\n/\nsum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 353, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Requests", "group": "Coprocessor Overview", "used_for": "Used for monitoring the total rate of coprocessor requests processed by TiKV instances. Helps identify overall coprocessor workload patterns and detect sudden increases in analytical query traffic or table scan operations.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22271, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Count", "group": "Scheduler", "used_for": "Used for monitoring the total number of tables being replicated by TiCDC's scheduler across changefeeds and keyspaces. Helps identify table distribution issues, verify changefeed configuration, and diagnose replication coverage problems when tables are unexpectedly missing or added.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 351, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the latency distribution of coprocessor requests in TiKV, which handle table scans and aggregations pushed down from TiDB. Essential for diagnosing slow query performance issues related to data scanning and identifying when coprocessor operations become bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 233, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Memory Usage of Storage Tasks", "group": "Disaggregated-Compute", "used_for": "Used for monitoring memory consumption of storage-related tasks in TiFlash disaggregated compute nodes, including query storage tasks, page fetching operations, and delta index cache. Essential for diagnosing memory pressure issues and optimizing resource allocation in TiFlash's storage layer when queries experience OOM errors or performance degradation.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_DT_DeltaIndexCacheSize{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingSharedColumnData{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 354, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Request Errors", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor request errors across TiKV instances. Essential for identifying data processing failures and diagnosing issues with complex queries, aggregations, or table scans that execute in TiKV's coprocessor layer.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 548, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Schedule Tasks", "group": "Scheduler", "used_for": "Used for monitoring the number of scheduled tasks in TiCDC's scheduler component, broken down by keyspace and changefeed. Helps identify task distribution imbalances and potential scheduling bottlenecks that could affect data replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed, scheduler, task, mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12041, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Encode and Send Checkpoint Message Duration", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of encoding and sending checkpoint messages to message queues (Kafka/Pulsar) in TiCDC's new architecture. Critical for diagnosing replication lag issues caused by slow checkpoint synchronization, which can impact downstream data consistency and recovery point objectives.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_checkpoint_ts_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_sink_mq_checkpoint_ts_message_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_sink_mq_checkpoint_ts_message_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1601, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "State", "group": "DR Autosync", "used_for": "Used for monitoring the current state of PD's disaster recovery (DR) auto-sync replication. Essential for verifying DR replication status and quickly identifying when the cluster transitions between sync states (e.g., sync, async, or disconnected) during failover scenarios or network issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 546, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Table Replication State", "group": "Scheduler", "used_for": "Used for monitoring the replication state distribution of tables across TiCDC changefeeds. Helps identify tables stuck in abnormal states (e.g., stopped, error) and diagnose replication pipeline issues when data synchronization stalls or fails.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed, state)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1606, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Recover progress", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) synchronization progress during cluster recovery operations in PD. Essential for tracking the completion status of DR auto-sync recovery and ensuring data consistency is restored after failover or disaster scenarios.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recover_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 352, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring coprocessor request latency in TiKV at p99 and p9999 percentiles. Critical for diagnosing slow analytical queries and table scan performance issues, as coprocessor handles complex operations like aggregations and filtering pushed down from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 179, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 OPS", "group": "S3", "used_for": "Used for monitoring the throughput of S3 operations (PUT, GET, HEAD) in TiFlash when using S3-compatible storage. Essential for diagnosing S3 storage performance issues, identifying bottlenecks in remote storage access, and understanding I/O patterns when TiFlash operates in disaggregated storage mode.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3PutDMFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeek{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22283, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Operator Count", "group": "Scheduler", "used_for": "Used for monitoring the count of different scheduler operations (add, move, split) in TiCDC's maintainer component. Helps diagnose changefeed scheduling behavior and identify potential issues with table distribution or rebalancing activities across TiCDC nodes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_maintainer_created_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", type=\"add\"}) by (keyspace_name,changefeed,mode)", "sum(ticdc_maintainer_created_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", type=\"move\"}) by (keyspace_name,changefeed,mode)", "sum(ticdc_maintainer_created_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", type=\"split\"}) by (keyspace_name,changefeed,mode)", "sum(ticdc_maintainer_created_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", type=\"merge\"}) by (keyspace_name,changefeed,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 182, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 Retry OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operation retries in TiFlash, including GetObject, PutObject, and PutDMFile operations. Essential for identifying S3 connectivity issues, network instability, or storage backend problems that cause TiFlash to retry S3 requests, which can impact query and write performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3PutDMFileRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOReadError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeekError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeekBackward{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1604, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "State ID", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) state identifier in PD's auto-sync replication. Helps track DR state transitions and verify the current replication state during failover scenarios or when diagnosing cross-region replication issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 545, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Span Count", "group": "Scheduler", "used_for": "Used for monitoring the total number of active spans being managed by TiCDC's scheduler across changefeeds. Helps identify scheduling bottlenecks, uneven workload distribution, and potential memory pressure issues when span counts grow unexpectedly high.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_span_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 343, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 HTTP OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 HTTP operations (read/write requests and errors) in TiFlash when using S3-compatible storage. Essential for diagnosing S3 storage performance issues, identifying request bottlenecks, and detecting connectivity or permission errors when TiFlash interacts with object storage.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3ReadRequestsCount{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteRequestsCount{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ReadRequestsErrors{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteRequestsErrors{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ReadRequestsThrottling{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteRequestsThrottling{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ReadRequestsRedirects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteRequestsRedirects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3ReadRequestsNotFound{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3WriteRequestsNotFound{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 180, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 Request Duration", "group": "S3", "used_for": "Used for monitoring S3 request latency in TiFlash at p99 and p9999 percentiles. Critical for diagnosing performance issues when TiFlash uses S3-compatible object storage, helping identify slow S3 operations that may impact query performance or data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type, $additional_groupby) / 1000000000)", "histogram_quantile(0.9999, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1612, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Memory Usage", "group": "GO Runtime", "used_for": "Used for monitoring memory consumption of PD (Placement Driver) processes, including resident memory, heap objects, and stack memory. Essential for diagnosing memory leaks, identifying OOM risks, and understanding PD's resource utilization patterns during cluster operations.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_heap_stacks_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_metadata_mcache_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}+go_memory_classes_metadata_mcache_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}+go_memory_classes_metadata_mspan_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}+go_memory_classes_metadata_mspan_inuse_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}+go_memory_classes_metadata_other_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_os_stacks_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}+go_memory_classes_other_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}+go_memory_classes_profiling_buckets_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 355, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of key scanning operations performed by TiKV's coprocessor during query execution. Essential for diagnosing performance issues related to table scans, identifying inefficient queries that scan excessive keys, and understanding I/O patterns in coprocessor requests.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 224, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Thread Pool Size", "group": "Pipeline Model", "used_for": "Used for monitoring the size of task thread pools in TiFlash's pipeline execution model. Essential for understanding thread resource allocation and diagnosing pipeline execution bottlenecks or thread pool saturation issues that may impact query processing performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\".*_task_thread_pool_size\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 359, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory Quota", "group": "Coprocessor Overview", "used_for": "Used for monitoring the memory quota allocated to TiKV's coprocessor component across instances. Helps identify memory allocation patterns and potential memory pressure issues when coprocessor requests consume excessive resources, which can impact query performance and cluster stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1613, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Estimated Live Objects", "group": "GO Runtime", "used_for": "Used for monitoring the number of live objects currently allocated in PD's Go heap memory. Helps diagnose memory leaks and excessive object allocation issues that could lead to increased garbage collection pressure and performance degradation.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 357, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total RocksDB Perf Statistics", "group": "Coprocessor Overview", "used_for": "Used for monitoring the aggregate rate of RocksDB internal operations during coprocessor query execution in TiKV. Helps diagnose storage-level performance bottlenecks and understand the internal workload characteristics when processing coprocessor requests, particularly useful for identifying excessive delete operations or other storage inefficiencies.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22284, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Operator Cost Duration", "group": "Scheduler", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC maintainer operator execution in the scheduler component. Critical for identifying performance bottlenecks in change data capture operations and diagnosing slow operator processing that could impact replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_maintainer_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,mode))", "sum(rate(ticdc_maintainer_finish_operators_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,mode) / \nsum(rate(ticdc_maintainer_finish_operators_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 22295, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Total Operator Count", "group": "Scheduler", "used_for": "Used for monitoring the total number of active operators in TiCDC's scheduler across changefeeds. Helps identify scheduling bottlenecks and abnormal operator accumulation that could indicate replication delays or resource contention issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_maintainer_total_operator_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\", type!=\"occupy\"}) by (keyspace_name,changefeed,type,mode)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 344, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 HTTP Request Duration", "group": "S3", "used_for": "Used for monitoring the tail latency (P99 and P99.99) of HTTP requests to S3 storage in TiFlash disaggregated storage architecture. Critical for diagnosing S3 access performance issues, network bottlenecks, or cloud storage service degradation that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_http_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type, $additional_groupby) / 1000000000)", "histogram_quantile(0.9999, sum(rate(tiflash_storage_s3_http_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_http_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type, $additional_groupby))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 341, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time transactions spend waiting to acquire latches in TiKV's scheduler, tracking both p99 and p9999 percentiles. Critical for diagnosing transaction contention issues and identifying hotspot keys that cause write conflicts, as high latch wait times indicate multiple transactions competing for the same resources.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1615, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "GC STW Duration (last 256 GC cycles)", "group": "GO Runtime", "used_for": "**Used For:**\n\nUsed for monitoring Go garbage collection stop-the-world (STW) pause durations in PD components across different percentiles (min, p50, p75, p99, max) over the last 256 GC cycles. Critical for diagnosing PD latency issues caused by excessive GC pauses, which can impact cluster scheduling decisions and TSO allocation performance.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"0\",component=~\".*(pd|tso|scheduling).*\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", quantile!~\"0|1\",component=~\".*(pd|tso|scheduling).*\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", quantile=\"1\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 359, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3RandomAccessFile OPS", "group": "S3", "used_for": "Used for monitoring S3 random access file operation rates in TiFlash, including read errors, seek errors, and backward seek operations. Critical for diagnosing S3 storage access issues and identifying abnormal file access patterns when TiFlash uses S3 as the storage backend.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3IOReadError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeekError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeekBackward{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)", "sum(rate(tiflash_system_profile_event_S3IOSeek{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type, $additional_groupby)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 360, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "S3 on-going instances", "group": "S3", "used_for": "Used for monitoring the current number of active S3 operations in TiFlash, including ongoing S3 requests and random access file operations. Helps identify S3 connection bottlenecks, detect resource leaks, and diagnose issues when TiFlash experiences slow data access or hangs due to S3 storage layer problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (type, $additional_groupby) (tiflash_system_current_metric_S3Requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})", "sum by (type, $additional_groupby) (tiflash_system_current_metric_S3RandomAccessFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1614, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "CPU Usage", "group": "GO Runtime", "used_for": "Used for monitoring CPU utilization of PD (Placement Driver) instances, including total process CPU usage, garbage collection CPU overhead, and configured CPU limits (maxprocs). Essential for identifying CPU bottlenecks, detecting abnormal CPU spikes, and ensuring PD has sufficient CPU resources for cluster scheduling and metadata management operations.", "component": "PD", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "(idelta((go_memstats_gc_cpu_fraction{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"} * (go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"} - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}) * tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"})[30s:]) > 0) / 15", "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 356, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per coprocessor request in TiKV, tracking both average and tail latencies (P99, P9999). Essential for diagnosing inefficient queries with excessive key scans, identifying full table scans, and detecting coprocessor performance bottlenecks that impact query execution efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 362, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Handle duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring coprocessor request handling latency across different TiKV stores, showing p99 and p9999 percentiles. Essential for identifying which specific TiKV instances have slow coprocessor performance and diagnosing uneven load distribution or storage-layer bottlenecks affecting analytical queries.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 541, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Slowest Table Checkpoint", "group": "Scheduler", "used_for": "Used for monitoring the checkpoint timestamp of the slowest table in TiCDC changefeeds, compared against the current PD cluster TSO. Critical for identifying replication lag bottlenecks caused by specific slow tables that are holding back the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1617, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Allocator Throughput", "group": "GO Runtime", "used_for": "Used for monitoring the memory allocation throughput of PD's Go runtime, including allocation rate, deallocation rate, and object creation rate. Essential for diagnosing memory pressure issues, identifying memory leaks, and understanding PD's memory allocation patterns during high load or performance degradation.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"})[30s:])", "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 544, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Slowest Table Replication State", "group": "Scheduler", "used_for": "Used for identifying tables experiencing replication delays in TiCDC's scheduler. Essential for diagnosing which specific tables are causing replication bottlenecks and understanding the distribution of slow replication states across changefeeds.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 358, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Response Size", "group": "Coprocessor Overview", "used_for": "Used for monitoring the total bytes per second of coprocessor responses returned from TiKV to TiDB. Helps identify network bandwidth usage and potential bottlenecks when large result sets are being transferred, especially during heavy analytical queries or table scans.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 228, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Status Change OPS", "group": "Pipeline Model", "used_for": "Used for monitoring the rate of pipeline task status transitions in TiFlash's pipeline execution model. Helps identify task scheduling patterns, execution bottlenecks, and abnormal task state changes that may indicate performance degradation or resource contention issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(tiflash_pipeline_task_change_to_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (type)\r\n" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 590, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Slowest Table Resolved Ts", "group": "Scheduler", "used_for": "Used for monitoring the resolved timestamp of the slowest table in TiCDC changefeeds, compared against the current PD TSO. Critical for identifying replication lag bottlenecks caused by specific slow tables that are holding back the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1616, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Goroutine Count", "group": "GO Runtime", "used_for": "Used for monitoring the number of goroutines and OS threads in PD instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate deadlocks or performance degradation in PD's scheduling and coordination functions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ " go_goroutines{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_threads{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 147, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "gRPC Message Count", "group": "TiKV", "used_for": "Used for monitoring the rate of gRPC message requests handled by TiKV instances, broken down by message type. Essential for identifying which gRPC operations are most active and diagnosing communication bottlenecks between TiCDC and TiKV during data replication.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 226, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Count", "group": "Pipeline Model", "used_for": "Used for monitoring the number of pipeline tasks in TiFlash's execution scheduler, broken down by instance and task type. Essential for diagnosing query execution bottlenecks, identifying task queue buildup, and understanding workload distribution across TiFlash nodes in the pipeline execution model.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\".*_tasks_count\"}) by (instance, type)", "sum(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\".*_tasks_count\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 363, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Wait duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time coprocessor requests spend waiting in TiKV's queue before execution, tracking p99 and p9999 percentiles. Critical for diagnosing coprocessor bottlenecks and identifying when TiKV is overloaded, causing queries to queue rather than execute immediately.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 543, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Slowest Table ID", "group": "Scheduler", "used_for": "Used for identifying the specific table ID experiencing the slowest replication performance in TiCDC's scheduler. Critical for pinpointing which table is causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues in specific changefeeds.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",keyspace_name=~\"$keyspace_name\", changefeed=~\"$changefeed\"}) by (keyspace_name,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 361, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Handle duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the latency of coprocessor request handling in TiKV at p99 and p9999 percentiles. Critical for diagnosing slow analytical queries and identifying performance bottlenecks in TiKV's coprocessor layer when processing pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 222, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Duration", "group": "Pipeline Model", "used_for": "Used for monitoring the p95 and p99 latency of pipeline task execution in TiFlash's pipeline execution model. Essential for identifying performance bottlenecks in TiFlash query processing and diagnosing slow analytical query issues caused by inefficient task scheduling or resource contention.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[2m]))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1618, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Estimated portion of CPU time", "group": "GO Runtime", "used_for": "Used for monitoring the estimated portion of CPU time spent on garbage collection in PD instances. Helps diagnose performance degradation caused by excessive GC overhead, particularly when investigating latency spikes or high CPU utilization in PD components.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(go_gc_heap_frees_by_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le,instance,component))", "increase(go_gc_heap_frees_by_size_bytes_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])/increase(go_gc_heap_frees_by_size_bytes_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_frees_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_frees_objects_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 368, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations in TiKV's coprocessor. Helps identify index scan throughput patterns and diagnose performance issues related to secondary index queries.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1621, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Golang GC", "group": "GO Runtime", "used_for": "Used for monitoring garbage collection frequency in PD's Go runtime, tracking automatic, forced, and total GC cycles per minute. Essential for diagnosing memory pressure issues and identifying when excessive GC activity impacts PD's scheduling and coordination performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(go_gc_cycles_automatic_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_cycles_forced_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "changes(go_gc_limiter_last_enabled_gc_cycle{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[30s])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1622, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Sync mutex wait", "group": "GO Runtime", "used_for": "Used for monitoring the rate of time spent waiting on mutex locks in PD's Go runtime. Helps identify synchronization bottlenecks and contention issues that can degrade PD performance and cluster coordination efficiency.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(go_sync_mutex_wait_total_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 367, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations in TiKV's coprocessor across different request types (select, select_by_index, etc.). Helps identify scan-heavy workloads and diagnose performance issues related to full table scans or index scans that may impact cluster throughput.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "CDC Network Traffic", "group": "TiKV", "used_for": "**Used For:**\n\nUsed for monitoring the network traffic rate of CDC (Change Data Capture) gRPC messages sent from TiKV instances. Essential for diagnosing CDC replication performance issues, identifying network bottlenecks, and tracking data transmission patterns between TiKV and downstream CDC consumers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 230, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Task Max Execute Time Per Round", "group": "Pipeline Model", "used_for": "**Used For:**\n\nUsed for monitoring the maximum execution time per scheduling round for TiFlash pipeline tasks at p95 and p99 percentiles. Critical for identifying pipeline scheduling inefficiencies, task execution bottlenecks, and unbalanced workload distribution that could impact query performance in TiFlash's pipeline execution model.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (le, type))\r", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m]))) by (le, type) / 1000000000)\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 366, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total DAG Executors", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) executor operations in TiKV's coprocessor. Helps identify coprocessor workload patterns and diagnose performance issues related to pushdown computation processing in TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 220, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of IO Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's IO task thread pool, which handles asynchronous I/O operations. Essential for identifying CPU bottlenecks in I/O processing and determining if the thread pool is saturated or underutilized during data read/write operations.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1619, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "GC STW Latency(>= go1.22.0)", "group": "GO Runtime", "used_for": "Used for monitoring the 99.9th percentile stop-the-world (STW) pause latency during garbage collection in PD's Go runtime (requires Go 1.22.0+). Critical for diagnosing GC-related performance issues that may cause PD request delays or scheduling jitter, especially when investigating unexpected latency spikes in cluster coordination operations.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,sum(rate(go_sched_pauses_total_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le,instance,component))", "rate(go_sched_pauses_total_gc_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m]) / rate(go_sched_pauses_total_gc_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "histogram_quantile(0.999,sum(rate(go_sched_pauses_total_other_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (instance,le))", "rate(go_sched_pauses_total_other_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])/rate(go_sched_pauses_total_other_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "histogram_quantile(0.999,sum(rate(go_sched_pauses_stopping_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le,instance,component))", "rate(go_sched_pauses_stopping_gc_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m]) / rate(go_sched_pauses_stopping_gc_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])", "histogram_quantile(0.999,sum(rate(go_sched_pauses_stopping_other_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (instance,le))", "rate(go_sched_pauses_stopping_other_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])/rate(go_sched_pauses_stopping_other_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 364, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Wait duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time coprocessor requests spend waiting in queue before execution across TiKV stores, tracking p99 and p9999 percentiles. Critical for diagnosing coprocessor bottlenecks and identifying stores experiencing thread pool saturation or resource contention that causes request queuing delays.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "CDC CPU", "group": "TiKV", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of CDC (Change Data Capture) related threads in TiKV instances, including CDC workers, CDC worker threads, and TSO threads. Essential for diagnosing CDC performance bottlenecks, identifying CPU-intensive replication operations, and detecting resource contention that may impact change data capture latency.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1620, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Goroutine scheduler latency", "group": "GO Runtime", "used_for": "Used for monitoring the tail latency of Go runtime's goroutine scheduler in PD instances, tracking extreme percentiles (p999 to p999999). Critical for diagnosing PD responsiveness issues caused by goroutine scheduling delays, which can impact cluster metadata operations and TSO allocation performance.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 999, 9999, 99999, 999999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le, instance,component))", "histogram_quantile(0.9999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le, instance,component))", "histogram_quantile(0.99999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le, instance,component))", "histogram_quantile(0.999999, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le, instance,component))", "histogram_quantile(1, sum(rate(go_sched_latencies_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le, instance,component))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 365, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total DAG Requests", "group": "Coprocessor Detail", "used_for": "Used for monitoring the total rate of DAG (Directed Acyclic Graph) coprocessor requests processed by TiKV instances. Essential for tracking coprocessor workload intensity and diagnosing issues when TiDB pushes computation down to TiKV storage nodes.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1624, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heap alloc", "group": "GO Runtime", "used_for": "Used for monitoring heap memory allocation patterns in PD (Placement Driver) components, including the 99.9th percentile allocation size and total allocation rate. Critical for diagnosing memory pressure issues, identifying memory leaks, and understanding PD's memory consumption behavior during cluster operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(go_gc_heap_allocs_by_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le,instance,component))", "increase(go_gc_heap_allocs_by_size_bytes_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])/increase(go_gc_heap_allocs_by_size_bytes_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_allocs_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_allocs_objects_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1623, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "GOGC & GOMEMLIMIT", "group": "GO Runtime", "used_for": "Used for monitoring PD's Go runtime memory management settings, specifically the GOGC percentage (garbage collection trigger threshold) and GOMEMLIMIT (soft memory limit). Essential for diagnosing memory pressure issues and understanding how Go's garbage collector is configured to manage PD's memory usage.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_gc_gogc_percent{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}", "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 369, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations across different RocksDB column families in TiKV's coprocessor. Helps diagnose which column families are experiencing high scan activity and identify potential hotspots or inefficient query patterns during table scans.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 300, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Wait notify task details", "group": "Pipeline Model", "used_for": "Used for monitoring the number of pipeline tasks waiting on notification events in TiFlash's pipeline execution model. Helps diagnose pipeline scheduling bottlenecks and coordination issues between different task types, particularly when tasks are blocked waiting for notifications from other components.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_pipeline_wait_on_notify_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance, type)", "sum(tiflash_pipeline_wait_on_notify_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 72, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Incremental Scan Duration Percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.99th percentile latency of TiCDC incremental scan operations on TiKV nodes. Critical for diagnosing tail latency issues in change data capture that could cause replication lag or affect downstream data synchronization performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))", "sum(rate(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / \nsum(rate(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 78, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Captured Region Count", "group": "TiKV", "used_for": "Used for monitoring the number of regions captured by TiCDC on each TiKV instance and their resolution status. Essential for diagnosing TiCDC replication issues, identifying regions stuck in capture or resolve phases, and ensuring data change events are being properly tracked across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 218, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of CPU Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's CPU task thread pool, which handles compute-intensive operations in the pipeline execution model. Essential for identifying CPU bottlenecks, thread pool saturation, and diagnosing query performance issues related to insufficient compute resources in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 194, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "CDC Memory Quota", "group": "TiKV", "used_for": "Used for monitoring CDC (Change Data Capture) memory usage against allocated quota in TiKV instances. Helps identify memory pressure in CDC sink operations and potential out-of-memory risks when replicating change events to downstream systems.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 216, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of Wait Reactor", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's Wait Reactor threads, which handle asynchronous I/O operations and event polling. Essential for diagnosing whether wait reactors are becoming CPU bottlenecks or if thread pool sizing needs adjustment for optimal async operation handling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22272, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Split Span Check Duration", "group": "Scheduler", "used_for": "Used for monitoring the 99.9th percentile latency of split span check operations in TiCDC's scheduler component. Critical for diagnosing performance bottlenecks in the maintainer's span splitting logic, which can impact changefeed replication efficiency and delay data synchronization.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_maintainer_split_span_check_duration_bucket{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,keyspace_name,changefeed,instance))", "sum(rate(ticdc_maintainer_split_span_check_duration_sum{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance) / \nsum(rate(ticdc_maintainer_split_span_check_duration_count{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (keyspace_name,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 246, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "TiFlash Resource Group", "group": "TiFlash Resource Control", "used_for": "Used for monitoring resource group quota management in TiFlash, tracking remaining tokens, average consumption speed, and total resource usage per group. Essential for diagnosing resource control effectiveness, identifying resource-constrained groups, and ensuring fair resource allocation across different workloads in TiFlash instances.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"remaining_tokens\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"avg_speed\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance,resource_group)", "sum(rate(tiflash_resource_group_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"total_consumption\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_fill_rate\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_capacity\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) by (instance,resource_group)", "sum(rate(tiflash_resource_group_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"request_gac_count\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,resource_group)", "sum(rate(tiflash_resource_group_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"gac_req_ru_consumption_delta\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,resource_group)", "sum(rate(tiflash_resource_group_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"compute_ru_consumption\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,resource_group)", "sum(rate(tiflash_resource_group_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_ru_consumption\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[2m])) by (instance,resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 68, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Initial Scan Duration", "group": "TiKV", "used_for": "Used for monitoring the duration of initial data scans performed by TiKV during TiCDC replication setup. Critical for diagnosing slow changefeed initialization and identifying performance bottlenecks when TiCDC starts capturing changes from existing data.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1625, "dashboard": "[SharedScope]PD-Cluster-Clinic", "title": "Heap free", "group": "GO Runtime", "used_for": "Used for monitoring the rate and distribution of heap memory being freed by Go's garbage collector in PD instances. Helps diagnose memory management efficiency and identify potential memory leak issues when freed memory rates are abnormally low or allocation patterns are unusual.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(go_gc_heap_frees_by_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\",component=~\".*(pd|tso|scheduling).*\"}[2m])) by (le,instance,component))", "increase(go_gc_heap_frees_by_size_bytes_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])/increase(go_gc_heap_frees_by_size_bytes_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_frees_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])", "increase(go_gc_heap_frees_objects_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\",component=~\".*(pd|tso|scheduling).*\"}[1m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 370, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations across different RocksDB column families in TiKV's coprocessor. Helps diagnose index scan performance issues and understand which column families are handling the most index lookup traffic during query execution.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query execution latency at multiple percentiles (p80, p95, p99, p999) across TiDB instances. Essential for identifying performance degradation, tail latency issues, and understanding query response time distribution when diagnosing slow query problems or user-reported latency concerns.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 140, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Initial Scan Tasks Status", "group": "TiKV", "used_for": "Used for monitoring the status of TiCDC initial scan tasks in TiKV, showing both ongoing and completed scan operations per instance. Essential for tracking CDC replication initialization progress and diagnosing slow or stalled initial data synchronization issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 338, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Status API Request Duration", "group": "Status Server", "used_for": "Used for monitoring the latency of status API requests in TiFlash's proxy server at P99 and P999 percentiles. Essential for diagnosing slow status queries and identifying performance degradation in TiFlash's status endpoint, which can impact cluster monitoring and health checks.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 99, 999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(0.999, \n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[$__rate_interval]\n)) by (le, path, $additional_groupby))", "histogram_quantile(0.99, \n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[$__rate_interval]\n)) by (le, path, $additional_groupby))", "(sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 371, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time spent checking memory locks during coprocessor operations in TiKV. Helps diagnose performance bottlenecks when coprocessor requests are delayed due to memory lock contention or validation overhead.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 372, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the latency of memory lock checking operations in TiKV's coprocessor during query execution. Critical for diagnosing performance bottlenecks when coprocessor requests experience delays due to in-memory lock verification, particularly useful when investigating slow queries with high p99 or p9999 latencies.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22297, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Incremental Scan Sink Duration Percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.99th percentile latency of TiCDC's incremental scan sink operations in TiKV. Critical for identifying extreme tail latency in change data capture processing, helping diagnose replication lag and data synchronization bottlenecks when TiCDC pulls incremental changes from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_sink_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))", "sum(rate(tikv_cdc_scan_sink_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / \nsum(rate(tikv_cdc_scan_sink_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 201, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Request Unit", "group": "TiFlash Resource Control", "used_for": "Used for monitoring Request Unit (RU) consumption in TiFlash across storage replica synchronization and compute operations. Helps track resource usage patterns by keyspace/cluster, identify resource-intensive workloads, and diagnose issues related to TiDB's resource control quotas in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "additional_groupby", "tiflash_role" ], "promql": [ "sum(rate(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[2m])) by (keyspace_id, $additional_groupby)", "sum(increase(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[24h])) by (keyspace_id, $additional_groupby)", "sum(rate(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[2m])) by (cluster_id, $additional_groupby)", "sum(increase(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[24h])) by (cluster_id, $additional_groupby)", "sum(rate(tiflash_storage_ru_read_bytes{instance=~\"$tiflash_role\"}[2m])) by (keyspace, resource_group, type, $additional_groupby) / (64 * 1024)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 42, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (OK vs Error). Essential for tracking overall query throughput, identifying error spikes, and comparing current traffic patterns against historical baselines (1-day offset) to detect anomalies or performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 376, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Poll Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the average time spent polling tasks in TiKV's unified read pool. Critical for diagnosing read performance degradation and identifying whether tasks are taking too long to execute, which could indicate resource contention or inefficient query processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "((\n tikv_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 139, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "CDC Total Scan Bytes", "group": "TiKV", "used_for": "Used for monitoring the total bytes scanned by TiCDC (Change Data Capture) on each TiKV instance. Essential for tracking CDC workload intensity, identifying instances with heavy scan operations, and diagnosing replication lag or performance bottlenecks in change data capture processes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 341, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Status API Request (op/s)", "group": "Status Server", "used_for": "Used for monitoring the rate of status API requests handled by TiFlash's proxy server, measured in operations per second. Helps identify unusual API traffic patterns, potential monitoring system issues, or excessive status checks that could impact TiFlash performance.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "proxy_instance", "tidb_cluster", "tiflash_role" ], "promql": [ "sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 374, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Schedule Wait Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the time tasks spend waiting in the queue before being scheduled in TiKV's unified read pool. Critical for diagnosing read performance bottlenecks caused by thread pool saturation or scheduling delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 286, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "In-Memory Vector Index Instances", "group": "Vector Search", "used_for": "Used for monitoring the number of active vector index instances loaded in TiFlash memory across different instances and types. Essential for tracking memory resource utilization of vector search features and identifying potential memory pressure from vector index operations.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "sum by (type, instance) (\n tiflash_vector_index_active_instances{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\", instance=~\"$tiflash_role\"\n }\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 137, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Failed Query OPM", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per minute in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing issues like syntax errors, permission problems, or resource constraints affecting query success rates.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 134, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "99 Duration", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query execution duration in TiDB instances. Essential for identifying performance degradation and detecting slow queries that affect the majority of users, helping diagnose latency issues before they impact overall cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 267, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Affected Rows By Type", "group": "Query Summary", "used_for": "Used for monitoring the rate of rows affected by different SQL statement types (INSERT, UPDATE, DELETE, etc.) in TiDB. Helps identify write-heavy workloads, understand data modification patterns, and diagnose unexpected bulk operations or data changes.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (sql_type)", "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 21, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for understanding query workload patterns, identifying traffic spikes, and diagnosing performance issues related to query throughput changes.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 379, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "First Poll Count", "group": "Unified Read Pool", "used_for": "Used for monitoring the rate of first-time task polls in TiKV's unified read pool. Helps diagnose task scheduling efficiency and identify whether the read pool is processing new tasks promptly or experiencing queuing delays.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "99.9% Vector Search Duration (Per Request)", "group": "Vector Search", "used_for": "Used for monitoring the 99.9th percentile latency of vector search operations in TiFlash per request. Critical for identifying tail latency issues in vector similarity searches and diagnosing performance degradation affecting the slowest vector queries.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(\n 0.999, \n sum(rate(\n tiflash_vector_index_duration_bucket{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\", instance=~\"$tiflash_role\", \n type!=\"build\"\n }\n [$__rate_interval]\n )) by (le, type)\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring the Commands Per Second (CPS) rate for each TiDB instance, showing the total query execution rate broken down by individual servers. Essential for identifying load distribution imbalances across TiDB instances and detecting if specific instances are handling disproportionate query traffic.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 288, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "Vector Index Estimated Memory Usage", "group": "Vector Search", "used_for": "Used for monitoring the estimated memory consumption of vector indexes in TiFlash, including both index memory usage and file-backed RSS memory. Essential for capacity planning and diagnosing memory pressure issues when running vector search workloads, helping identify if vector indexes are consuming excessive memory resources.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "tiflash_vector_index_memory_usage{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\", instance=~\"$tiflash_role\"\n}", "tiflash_process_rss_by_type_bytes{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\", instance=~\"$tiflash_role\",\n type=\"file\"\n}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 152, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Min Resolved Ts", "group": "TiKV", "used_for": "Used for monitoring the replication lag between TiKV's resolved timestamp and the current cluster TSO in TiCDC. Critical for identifying data replication delays and diagnosing CDC pipeline bottlenecks that may prevent downstream systems from receiving timely updates.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\"})", "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 378, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Idle Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the average idle time of worker threads in TiKV's unified read pool. Helps identify thread pool efficiency issues—high idle duration may indicate over-provisioned threads, while consistently low values suggest the pool is saturated and may need scaling.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 153, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Min Resolved Region", "group": "TiKV", "used_for": "Used for monitoring the minimum resolved timestamp region in TiKV instances for CDC (Change Data Capture). Helps identify replication lag bottlenecks by showing which TiKV instance has the slowest region preventing CDC from advancing its checkpoint.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 132, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "95 Duration", "group": "Query Summary", "used_for": "Used for monitoring the 95th percentile query execution latency in TiDB. Helps identify performance degradation affecting most users and diagnose whether query slowdowns are widespread or limited to specific outliers.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 284, "dashboard": "[SharedScope]TiFlash-Summary-Cluster-Clinic", "title": "99.9% Vector Index Build Duration (Per DMFile Column)", "group": "Vector Search", "used_for": "Used for monitoring the 99.9th percentile latency of vector index building operations per DMFile column in TiFlash. Critical for diagnosing performance bottlenecks in vector search index construction and identifying slow index build operations that may impact query readiness.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster", "tiflash_role" ], "promql": [ "histogram_quantile(\n 0.999, \n sum(rate(\n tiflash_vector_index_duration_bucket{\n k8s_cluster=\"$k8s_cluster\", \n tidb_cluster_id=\"$tidb_cluster\", \n instance=~\"$instance\", instance=~\"$tiflash_role\", \n type=\"build\"\n }\n [$__rate_interval]\n )) by (le, type)\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 146, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Old Value Seek Duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV spends seeking old values during CDC (Change Data Capture) operations. Critical for diagnosing CDC replication performance issues and identifying bottlenecks in historical data retrieval for change feeds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 136, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "999 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile query execution duration in TiDB, showing the worst-case latency experienced by 0.1% of queries. Critical for identifying tail latency issues, slow query outliers, and ensuring consistent query performance meets SLA requirements.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Incremental Scan Speed", "group": "TiKV", "used_for": "Used for monitoring the rate of data scanning (bytes per second) during TiCDC's incremental scan phase on each TiKV instance. Helps identify scanning performance bottlenecks and uneven data distribution when TiCDC is catching up with change data capture workloads.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 375, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Running Tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiKV's unified read pool. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient thread resources or slow query processing.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22298, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Incremental Scan Disk Speed", "group": "TiKV", "used_for": "Used for monitoring the disk read throughput (bytes per second) during TiCDC's incremental scan operations on each TiKV instance. Essential for diagnosing TiCDC replication performance issues and identifying disk I/O bottlenecks that may slow down change data capture.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_disk_read_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration 95 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 95th percentile query execution latency for individual TiDB instances. Helps identify performance variations across instances and pinpoint specific TiDB nodes experiencing elevated query latency issues.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 143, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Old Value Cache Hit", "group": "TiKV", "used_for": "Used for monitoring the cache hit rate of TiCDC's old value cache in TiKV, which stores historical row values needed for change data capture. Essential for diagnosing TiCDC performance issues, as low hit rates indicate insufficient cache sizing and can lead to increased disk I/O and replication latency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 70, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Resolved Ts Lag Duration Percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.999th percentile lag between TiKV's resolved timestamp and current time in TiCDC replication. Critical for identifying extreme tail latency in CDC data synchronization and diagnosing replication delays that could impact downstream data freshness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 381, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC tasks", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) tasks in TiKV, including total GC tasks executed, skipped tasks, and failed tasks. Essential for diagnosing GC performance issues, identifying when GC operations are being skipped or failing, which can lead to storage bloat and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 130, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "80 Duration", "group": "Query Summary", "used_for": "Used for monitoring the 80th percentile query execution latency in TiDB. Helps identify typical query performance and detect degradation affecting the majority of queries, serving as an early indicator before p99 latency issues become severe.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 383, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TiDB GC seconds", "group": "GC", "used_for": "Used for monitoring the latency distribution of TiDB's garbage collection operations across different GC stages. Essential for diagnosing GC performance issues that could impact transaction visibility and storage space reclamation efficiency.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, stage, keyspace_name, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 385, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "ResolveLocks Progress", "group": "GC", "used_for": "Used for monitoring the progress and results of lock resolution operations during garbage collection in TiKV. Essential for diagnosing GC efficiency issues and identifying problems with cleaning up stale transaction locks that could block new transactions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, type, keyspace_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration 80 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 80th percentile query execution latency for individual TiDB instances. Helps identify performance variations across instances and pinpoint specific TiDB nodes experiencing elevated query latency that may not be visible in cluster-wide averages.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 384, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TiDB GC worker actions", "group": "GC", "used_for": "Used for monitoring the rate of TiDB garbage collection worker actions by type and keyspace. Helps diagnose GC performance issues, identify bottlenecks in different GC phases (like resolve locks, delete ranges), and track GC activity patterns across keyspaces.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, keyspace_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 218, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Connection Idle Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the duration that TiDB connections remain idle between queries at p80, p90, and p99 percentiles. Helps identify connection pool efficiency issues, detect applications holding connections unnecessarily long, and diagnose potential connection leak problems that could lead to connection exhaustion.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 25, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration 99 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 99th percentile query execution duration for individual TiDB instances. Helps identify performance outliers and pinpoint which specific TiDB nodes are experiencing elevated query latency, enabling targeted troubleshooting of instance-level bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 141, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Old Value Seek Operation", "group": "TiKV", "used_for": "Used for monitoring the rate of old value scan operations in TiKV when TiCDC retrieves historical data for change data capture. Essential for diagnosing TiCDC performance issues related to old value lookups and identifying I/O bottlenecks in the column family storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, cf, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 387, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC speed", "group": "GC", "used_for": "Used for monitoring the rate at which TiKV's garbage collection (GC) process deletes obsolete MVCC versions. Essential for evaluating GC efficiency and diagnosing issues when deleted data isn't being cleaned up quickly enough, which can lead to storage bloat and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 145, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Old Value Cache Size", "group": "TiKV", "used_for": "Used for monitoring TiCDC's old value cache in TiKV, tracking the number of cached entries, memory usage, and memory quota per instance. Essential for diagnosing TiCDC replication performance issues and detecting memory pressure when capturing historical row values for change data capture.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10023, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Fsync Duration", "group": "Redo", "used_for": "Used for monitoring the fsync operation latency in TiCDC's redo log system. Critical for diagnosing redo log write performance issues and identifying disk I/O bottlenecks that could impact data replication reliability and changefeed lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10025, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Flushall Duration", "group": "Redo", "used_for": "Used for monitoring the duration of redo log flush-all operations in TiCDC's new architecture. Critical for diagnosing redo log write performance bottlenecks and identifying when batch flush operations are taking longer than expected, which can impact replication latency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572003, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Essential for identifying recent restarts, detecting instance instability, and verifying cluster availability during troubleshooting or after maintenance operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10029, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Flush Log Duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log flush operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery mechanism, especially when replication lag increases or redo log writes become slow.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 386, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC Progress", "group": "GC", "used_for": "Used for monitoring the progress of TiKV's automatic garbage collection process by tracking the number of regions processed during GC scans. Helps operators verify that GC is running properly and diagnose issues when old MVCC versions aren't being cleaned up as expected.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 393, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC scan default details", "group": "GC", "used_for": "Used for monitoring the rate of keys scanned by the garbage collection worker in TiKV's default column family. Helps diagnose GC performance issues and understand the workload intensity of cleaning up obsolete MVCC versions in the default storage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10031, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Write Rows / s", "group": "Redo", "used_for": "Used for monitoring the rate of rows written to redo logs per second in TiCDC, broken down by instance and changefeed. Essential for tracking redo log write throughput and identifying performance bottlenecks or imbalances in change data capture workloads.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[2m])) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 142, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Old Value Seek Duration", "group": "TiKV", "used_for": "Used for monitoring the 99.9th percentile latency of old value seek operations in TiKV for TiCDC change data capture. Critical for diagnosing performance bottlenecks when TiCDC retrieves historical row values during replication, especially when experiencing CDC lag or slow data synchronization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 391, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC in Compaction Filter", "group": "GC", "used_for": "Used for monitoring garbage collection activity during RocksDB compaction in TiKV, tracking filtered keys, skipped operations, and MVCC rollbacks. Essential for diagnosing GC efficiency issues and understanding how much obsolete data is being cleaned up during background compaction processes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 382, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC tasks duration", "group": "GC", "used_for": "Used for monitoring the latency of garbage collection tasks in TiKV, tracking both p99 and p9999 percentiles. Essential for diagnosing GC performance issues that could impact MVCC version cleanup and storage space reclamation efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "command", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10027, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Write Log Duration", "group": "Redo", "used_for": "Used for monitoring the time TiCDC spends writing redo logs to storage, which is critical for disaster recovery scenarios. Helps diagnose performance bottlenecks in the redo log writing path that could impact changefeed replication latency and recovery point objectives (RPO).", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10033, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Redo Write Bytes / s", "group": "Redo", "used_for": "Used for monitoring the redo log write throughput (bytes per second) for each TiCDC instance. Essential for diagnosing redo log I/O performance issues, identifying write bottlenecks, and ensuring sufficient disk bandwidth for change data capture operations.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 68, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Internal SQL OPS", "group": "Query Detail", "used_for": "Used for monitoring the rate of internal SQL operations executed by TiDB, such as system queries for statistics collection, background jobs, and internal maintenance tasks. Helps distinguish internal database overhead from user workload and diagnose performance issues caused by excessive internal operations.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 168, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiDB server instances, showing both actual CPU usage rate and the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation, and determining if TiDB instances need vertical scaling or load balancing.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 392, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC scan write details", "group": "GC", "used_for": "Used for monitoring the rate of keys scanned during garbage collection in TiKV's write column family. Essential for diagnosing GC performance issues and understanding the workload intensity of cleaning up obsolete MVCC versions in the write CF.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10035, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Worker Busy Ratio", "group": "Redo", "used_for": "Used for monitoring the busy ratio of redo log workers in TiCDC, indicating how much time workers spend actively processing versus idle. High ratios suggest workers are saturated and may need scaling, while consistently low ratios indicate underutilization or potential I/O bottlenecks in the redo log pipeline.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])/10) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 184, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiDB Server Status", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the availability status of TiDB server instances in the cluster. Displays the count of online (up) and offline (down) TiDB nodes, helping quickly identify server failures or unexpected instance shutdowns that could impact cluster availability.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} == 1)", "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} == 0)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 389, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC lifetime", "group": "GC", "used_for": "Used for monitoring the configured GC (Garbage Collection) lifetime setting in TiKV, which determines how long historical MVCC versions are retained. Essential for diagnosing issues related to storage space growth, transaction conflicts with long-running queries, and ensuring the GC configuration aligns with business requirements for historical data access.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) by (keyspace_name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 188, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiDB server processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause service disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 157, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Memory Usage", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC runtime memory consumption, including resident memory, heap allocation, and garbage collection thresholds. Essential for diagnosing memory leaks, OOM issues, and optimizing GC behavior in TiCDC change data capture processes.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 395, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Lock Manager Thread CPU", "group": "Pessimistic Locking", "used_for": "Used for monitoring CPU usage of TiKV's lock manager threads, including waiter manager and deadlock detector threads. Critical for diagnosing performance issues in pessimistic transaction locking, especially when experiencing lock contention or deadlock detection overhead.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiDB instances. Essential for detecting goroutine leaks, resource exhaustion issues, and identifying abnormal concurrency patterns that could indicate connection pooling problems or stuck operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_sched_goroutines_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\"tidb.*\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 292, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Runtime GC Rate And GOMEMLIMIT", "group": "Server", "used_for": "Used for monitoring TiDB's Go runtime garbage collection frequency and memory limit (GOMEMLIMIT). Essential for diagnosing memory pressure issues, tuning GC behavior, and understanding whether TiDB instances are approaching their configured memory limits, which can impact query performance and stability.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 398, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Lock Waiting Queue", "group": "Pessimistic Locking", "used_for": "Used for monitoring the number of transactions waiting in TiKV's pessimistic lock queue. Critical for diagnosing lock contention issues, identifying when transactions are blocked waiting for locks, and detecting potential deadlock or high-concurrency scenarios that impact transaction throughput.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) ", "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10001, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DDL Schema Count", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of DDL schema changes published to Pulsar topics by TiCDC. Essential for tracking schema evolution events and diagnosing issues with DDL replication to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 160, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "GC STW Duration (last 256 GC cycles)", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring Go garbage collection stop-the-world (STW) pause durations in TiCDC runtime instances over the last 256 GC cycles. Essential for diagnosing performance issues caused by excessive GC pauses, which can lead to replication lag and increased latency in change data capture operations.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 272, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Queries In Multi-Statement", "group": "Query Detail", "used_for": "Used for monitoring the average number of queries executed within multi-statement batches in TiDB. Helps identify applications using multi-statement features and assess the efficiency of batch query execution patterns.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))/sum(rate(tidb_server_multi_query_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 396, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Lock Manager Handled tasks", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of tasks handled by TiKV's lock manager in pessimistic locking scenarios. Essential for diagnosing lock contention issues, deadlock detection performance, and understanding the workload on the distributed lock management system.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22470, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Memory Quota", "group": "Redo", "used_for": "Used for monitoring memory quota allocation and usage in TiCDC's redo log system. Helps identify memory pressure issues and potential out-of-memory risks in the change data capture pipeline, particularly when redo logging is enabled.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "keyspace_name", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", area=~\"$changefeed\", instance=~\"$ticdc_instance\", module=~\"event-collector-redo\"}) by (keyspace_name, area, instance, module, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "Used for monitoring the number of active client connections to TiDB instances and tracking connection token usage. Essential for diagnosing connection exhaustion issues, identifying connection leaks, and understanding client connection patterns across the cluster.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 377, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Scheduled Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the average time tasks wait in the queue before being executed in TiKV's unified read pool. Critical for identifying thread pool saturation and scheduling delays that can cause read request latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration 999 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 99.9th percentile query execution duration for individual TiDB instances. Essential for identifying which specific TiDB nodes are experiencing tail latency issues or performance degradation, enabling targeted troubleshooting of slow query problems at the instance level.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10003, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DDL Schema Fail", "group": "Pulsar Sink", "used_for": "Used for monitoring DDL schema publication failures when TiCDC replicates data to Pulsar sink. Critical for diagnosing schema synchronization issues and identifying when DDL changes fail to be published to Pulsar topics, which can block downstream data replication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 94, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Failed Query OPM Detail", "group": "Query Detail", "used_for": "Used for monitoring the rate of failed query executions per minute in TiDB instances, broken down by error type. Essential for identifying query failure patterns, diagnosing application errors, and tracking the health of SQL execution across the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 205, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Disconnection Count", "group": "Server", "used_for": "Used for monitoring the total number of client disconnections from TiDB instances, categorized by disconnection result type. Essential for diagnosing connection stability issues, identifying abnormal client disconnection patterns, and troubleshooting network or client-side problems affecting database connectivity.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (instance, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 400, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Detect error", "group": "Pessimistic Locking", "used_for": "Used for monitoring the error rate in TiKV's pessimistic lock manager. Critical for detecting deadlock detection failures, lock timeout issues, and other locking mechanism errors that could impact transaction processing and data consistency.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 397, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Waiter lifetime duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring how long transactions wait for locks to be released in TiKV's pessimistic locking system, tracking p99 and p9999 percentiles. Critical for diagnosing lock contention issues and identifying transactions experiencing excessive wait times due to conflicting pessimistic locks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10004, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DML Schema Count", "group": "Pulsar Sink", "used_for": "Used for monitoring the rate of DML schema changes published to Pulsar topics by TiCDC. Helps identify schema evolution frequency and diagnose issues with DDL propagation or unexpected schema modification patterns in the data replication pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 401, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Deadlock detector leader", "group": "Pessimistic Locking", "used_for": "Used for identifying which TiKV instance is currently serving as the deadlock detector leader in pessimistic locking scenarios. Essential for troubleshooting deadlock detection issues and verifying proper leader election when investigating transaction conflicts or lock wait timeouts.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 49, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Events OPM", "group": "Server", "used_for": "Used for monitoring the rate of server events occurring in TiDB instances over 10-minute intervals. Helps track operational events like connection changes, schema updates, and internal state transitions to identify unusual activity patterns or potential issues in the TiDB server layer.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[10m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10005, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DML Schema Success", "group": "Pulsar Sink", "used_for": "**Used For:**\n\nUsed for monitoring the successful publication rate of DML schema changes to Pulsar topics in TiCDC. Helps verify that schema synchronization is working correctly and diagnose issues when database schema changes fail to propagate to downstream Pulsar consumers.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 165, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Prepare Statement Count", "group": "Server", "used_for": "Used for monitoring the number of prepared statements currently cached in each TiDB instance and across the cluster. Helps identify potential memory pressure from excessive prepared statement accumulation and diagnose connection pooling or statement lifecycle management issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}", "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 158, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Estimated Live Objects", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring the number of live objects currently allocated in the Go heap memory of TiCDC runtime instances. Helps diagnose memory management issues, potential memory leaks, and assess the memory pressure from object allocation patterns in TiCDC's new architecture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 161, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Allocator Throughput", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring the memory allocation throughput in TiCDC runtime instances, tracking the rate of heap allocations, deallocations, and total malloc operations. Essential for diagnosing memory pressure issues, identifying memory leaks, and understanding the memory allocation patterns that may impact TiCDC's replication performance.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 399, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Deadlock detect duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of deadlock detection operations in TiKV's pessimistic locking system, showing p99 and p9999 percentiles. Critical for diagnosing performance issues with pessimistic transactions, especially when applications experience lock wait timeouts or transaction conflicts that trigger deadlock detection.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 111, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Get Token Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of token acquisition in TiDB's resource control system. Critical for diagnosing resource group throttling issues and identifying when applications experience delays waiting for execution tokens.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 404, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Pessimistic lock activities", "group": "Pessimistic Locking", "used_for": "Used for monitoring the volume and types of pessimistic lock operations in TiKV, including lock wait queue entries and lock acquisition activities. Essential for diagnosing transaction contention issues, identifying lock conflicts, and troubleshooting scenarios where transactions are waiting for locks or experiencing deadlocks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10007, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Bytes Published", "group": "Pulsar Sink", "used_for": "Used for monitoring the total bytes published by TiCDC's Pulsar client to the Pulsar message queue, broken down by changefeed and instance. Essential for tracking data throughput to Pulsar sink and diagnosing replication bandwidth issues or bottlenecks in the CDC pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 82, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Keep Alive OPM", "group": "Server", "used_for": "Used for monitoring the rate of keep-alive operations per minute sent by TiDB instances to maintain connections with clients or other components. Helps identify connection health issues, network instability, or abnormal connection patterns that could indicate client timeout problems or network disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10002, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DDL Schema Success", "group": "Pulsar Sink", "used_for": "**Used For:**\n\nUsed for monitoring the successful publication rate of DDL schema changes to Pulsar topics in TiCDC's new architecture. Essential for verifying that schema evolution events are being correctly propagated to downstream Pulsar consumers and diagnosing DDL replication failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10006, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Published DML Schema Fail", "group": "Pulsar Sink", "used_for": "Used for monitoring failed DML schema publications to Pulsar topics in TiCDC's Pulsar sink. Critical for identifying schema synchronization issues that could cause data replication failures or inconsistencies between TiDB and downstream Pulsar consumers.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10008, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Connections Opened", "group": "Pulsar Sink", "used_for": "**Used For:**\n\nUsed for monitoring the number of active Pulsar client connections opened by TiCDC changefeeds. Helps diagnose connection management issues, detect connection leaks, and verify proper connection pooling behavior when replicating data to Pulsar sinks.", "component": "TICDC", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 402, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total pessimistic locks memory size", "group": "Pessimistic Locking", "used_for": "Used for monitoring the total memory consumed by pessimistic locks in TiKV instances. Critical for identifying memory pressure from lock accumulation, which can indicate lock contention issues, long-running transactions, or potential memory leaks in pessimistic transaction scenarios.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 403, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "In-memory pessimistic locking result", "group": "Pessimistic Locking", "used_for": "Used for monitoring the success and failure rates of in-memory pessimistic lock operations in TiKV. Helps diagnose pessimistic transaction locking issues and identify when locks cannot be acquired in memory, which may indicate memory pressure or lock contention problems.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 405, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Lengths of lock wait queues when transaction enqueues", "group": "Pessimistic Locking", "used_for": "Used for monitoring the distribution of lock wait queue lengths when transactions enqueue for pessimistic locks in TiKV. Helps diagnose lock contention issues by showing how many transactions are waiting for locks, indicating potential hotspot problems or excessive concurrent access to the same keys.", "component": "TIKV", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Panic And Critial Error", "group": "Server", "used_for": "Used for monitoring panic events and critical errors occurring in TiDB server instances. Essential for detecting severe system failures and stability issues that require immediate attention, as these errors typically indicate serious problems that could impact cluster availability and data integrity.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])", "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10009, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client RPC Count", "group": "Pulsar Sink", "used_for": "**Used For:**\n\nUsed for monitoring the total count of RPC calls made by TiCDC's Pulsar client to the Pulsar cluster, broken down by changefeed and instance. Helps diagnose communication issues between TiCDC and Pulsar, identify abnormal RPC patterns, and troubleshoot connectivity or performance problems in the Pulsar sink pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 388, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TiKV Side GC Safe Points", "group": "GC", "used_for": "Used for monitoring the GC (Garbage Collection) safe point timestamps across TiKV instances, which determines the oldest data version that must be retained. Critical for diagnosing issues with MVCC data cleanup, identifying stale data accumulation, and troubleshooting problems where GC cannot advance due to long-running transactions or snapshot reads.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) != 0", "max((\n tikv_raftstore_keyspace_gc_safe_points\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n \n)) by (instance, keyspace_name, $additional_groupby) / (2^18) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 406, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "In-memory scan lock read duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of in-memory lock scanning operations during pessimistic transaction processing in TiKV. Critical for diagnosing performance bottlenecks in pessimistic locking scenarios, especially when transactions need to check existing locks before acquiring new ones.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 211, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Client Data Traffic", "group": "Server", "used_for": "Used for monitoring the network traffic volume between TiDB servers and clients, broken down by packet type (read/write). Essential for identifying network bandwidth bottlenecks, unusual traffic patterns, and diagnosing client-server communication issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 390, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC interval", "group": "GC", "used_for": "Used for monitoring the configured garbage collection (GC) interval in TiKV, which determines how frequently GC operations run to clean up obsolete MVCC versions. Essential for understanding GC scheduling behavior and diagnosing issues related to storage space accumulation or GC frequency tuning.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) by (keyspace_name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 411, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks queued in TiKV's FuturePool thread pools. Helps identify thread pool saturation and task backlog issues that can lead to request processing delays and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 408, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiKV worker threads across different worker types (e.g., scheduler, raftstore, apply). Essential for diagnosing task processing bottlenecks, identifying overloaded workers, and understanding the distribution of internal workload across TiKV's thread pools.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 166, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Time Jump Back OPS", "group": "Server", "used_for": "Used for monitoring the rate of system time backward jumps detected by TiDB servers. Critical for identifying clock synchronization issues that can cause transaction timestamp anomalies, data consistency problems, and TSO (Timestamp Oracle) conflicts in distributed transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 410, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiKV's FuturePool thread pools. Helps identify task processing throughput and potential bottlenecks in asynchronous task execution across TiKV instances.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 409, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks queued in TiKV's various worker threads (such as scheduler, apply, and raftstore workers). Essential for identifying thread pool saturation and task processing bottlenecks that can lead to request timeouts and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10010, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Producer Latency", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC's Pulsar client when producing messages to Pulsar topics. Critical for identifying tail latency issues in the data replication pipeline and diagnosing slow message delivery that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r", "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10011, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Producer RPC Latency", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile RPC latency of Pulsar client producers in TiCDC's new architecture. Critical for diagnosing slow data replication to Pulsar and identifying network or Pulsar broker performance bottlenecks affecting change data capture delivery.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 167, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Handshake Error OPS", "group": "Server", "used_for": "Used for monitoring the rate of client connection handshake failures per TiDB instance. Essential for diagnosing authentication issues, SSL/TLS configuration problems, or client compatibility errors that prevent successful database connections.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572020, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Active Users", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of currently active database users connected to TiDB servers. Helps identify connection usage patterns, detect potential connection leaks, and diagnose issues related to excessive concurrent user sessions that may impact cluster performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_active_users{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 413, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of requests sent from TiKV instances to PD (Placement Driver). Helps identify communication patterns between TiKV and PD, and diagnose issues when TiKV cannot reach PD or experiences high request volumes for metadata operations like region heartbeats and TSO requests.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10012, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Producer Pending Messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in the Pulsar producer queue for TiCDC changefeeds. Critical for identifying message backlog issues and potential bottlenecks in data replication to Pulsar, which could indicate downstream processing delays or producer capacity problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10013, "dashboard": "[SharedScope]Prometheus-TiCDC-New-Arch-Clinic", "title": "Pulsar Client Producer Pending Messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in the Pulsar producer queue for TiCDC changefeeds. Helps identify message backlog issues and potential bottlenecks in data replication to Pulsar, indicating when the producer cannot keep up with the change data capture rate.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 191, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Skip Binlog Count", "group": "Server", "used_for": "Used for monitoring the total count of critical errors related to binlog skipping in TiDB instances. Essential for identifying binlog replication failures or data consistency issues when TiDB cannot write binlog entries, which could impact downstream replication to TiDB Binlog or TiCDC.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 171, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "RCCheckTS WriteConflict Num", "group": "Server", "used_for": "Used for monitoring the number of write conflicts detected during read-committed isolation level timestamp checks in TiDB. Helps diagnose transaction contention issues and concurrent write conflicts that may cause transaction retries or failures.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_rc_check_ts_conflict_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 21, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "QPS", "group": "", "used_for": "Used for monitoring the total queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, etc.). Essential for tracking overall database throughput, identifying traffic patterns, and detecting sudden load changes or anomalies in query workload distribution.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 416, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiKV and PD (Placement Driver). Helps diagnose cluster metadata synchronization issues and identify problems with region peer state validation that could affect data placement and replication.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 414, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of TiKV requests to PD (Placement Driver). Helps identify communication delays between TiKV and PD, which can indicate network issues, PD overload, or cluster coordination problems affecting metadata operations and region scheduling.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring query execution latency at multiple percentiles (p80, p95, p99, p999) across TiDB instances. Essential for identifying performance degradation, tail latency issues, and understanding query response time distribution when diagnosing slow query problems or cluster performance bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 418, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD forward status", "group": "PD", "used_for": "Used for monitoring whether TiKV instances are forwarding PD requests to other PD nodes, indicating potential PD leader changes or network issues. Helps diagnose PD cluster instability and identify TiKV instances experiencing communication problems with the current PD leader.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572016, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Internal Sessions", "group": "Server", "used_for": "Used for monitoring the number of internal sessions in TiDB, which are system-level connections used for background tasks like statistics collection, auto-analyze, and internal operations. Helps identify resource consumption from internal processes and diagnose issues when internal session counts are abnormally high, potentially impacting cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_internal_sessions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 72, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Duration", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile transaction duration in TiDB, broken down by transaction mode (optimistic/pessimistic). Critical for identifying transaction latency issues, detecting slow commits, and diagnosing performance degradation in OLTP workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, txn_mode))", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", scope=~\"general\"}[2m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", scope=~\"general\"}[2m])) by (txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 69, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction OPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring transaction operations per second in TiDB, separated by general user transactions and internal system transactions. Essential for tracking transaction throughput, identifying workload patterns, and diagnosing performance issues when transaction processing rates deviate from expected levels.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (type, txn_mode)", "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"internal\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 42, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Command Per Second", "group": "", "used_for": "Used for monitoring the rate of SQL commands executed per second on TiDB servers, broken down by execution result (success/error). Essential for tracking overall database workload, identifying sudden traffic changes, comparing current performance against historical baselines (1-day offset), and detecting error rate spikes that may indicate application or database issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 136, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "999 Duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Critical for identifying tail latency issues and detecting slow query outliers that may impact user experience, especially when investigating performance degradation or timeout complaints.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 267, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Affected Rows By Type", "group": "", "used_for": "Used for monitoring the number of rows affected by different SQL statement types (INSERT, UPDATE, DELETE) in TiDB. Helps identify data modification patterns, detect unusual bulk operations, and diagnose potential issues with write-heavy workloads or runaway queries affecting large datasets.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type)", "sum(rate(tidb_executor_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 421, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Inspected duration per server", "group": "Slow Node Statistics", "used_for": "Used for monitoring the latency of disk and network inspection operations on individual TiKV servers. Helps identify slow nodes in the cluster by comparing inspection durations across instances, which is critical for diagnosing performance bottlenecks and uneven resource utilization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_disk_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_network_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, target, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "CPS By Instance", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the commands per second (CPS) rate for each TiDB instance in the cluster. Helps identify uneven query distribution across instances, detect instance-level performance anomalies, and diagnose whether specific TiDB servers are experiencing abnormal traffic patterns or load imbalances.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 137, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Failed Query OPM", "group": "", "used_for": "Used for monitoring the rate of failed queries per minute in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing which types of errors (e.g., syntax errors, permission issues, or execution timeouts) are affecting the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 419, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Pending TSO Requests", "group": "PD", "used_for": "Used for monitoring the number of TSO (Timestamp Oracle) requests waiting to be processed from TiKV to PD. Helps identify PD performance bottlenecks or network issues when TSO request queues build up, which can impact transaction processing latency across the cluster.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 112, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Slow Query", "group": "", "used_for": "Used for monitoring the 90th percentile latency breakdown of slow queries in TiDB, including total processing time, coprocessor execution time, and wait time. Essential for diagnosing performance bottlenecks in slow queries by identifying whether delays occur during query processing, TiKV coprocessor operations, or resource waiting.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 417, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD reconnection", "group": "PD", "used_for": "**Used For:**\n\nUsed for tracking the frequency of TiKV reconnections to PD (Placement Driver) servers. Essential for diagnosing network instability, PD availability issues, or connection problems that could impact cluster coordination and metadata operations.", "component": "TIKV", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Regions Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of region counts involved in transaction prewrite operations. Helps identify transactions that span many regions, which can indicate inefficient data access patterns or potential performance bottlenecks in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 36, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Session Retry Error OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of transaction retry errors in TiDB sessions, categorized by error type and SQL type. Essential for diagnosing transaction conflicts, lock timeouts, and other retry-related issues that impact application performance and data consistency.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type, sql_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 74, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Statement Num", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of SQL statements executed within transactions in TiDB. Helps identify transaction complexity patterns and diagnose issues related to large transactions that may cause performance degradation or lock contention.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 426, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot generation/apply wait duration $optional_quantile", "group": "Snapshot", "used_for": "Used for monitoring the wait time before Raft snapshot generation and application operations begin in TiKV. Critical for diagnosing delays in region snapshot processing, which can indicate scheduling bottlenecks or resource contention affecting region recovery and replication performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 196, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Commit Token Wait Duration", "group": "Transaction", "used_for": "Used for monitoring the time transactions spend waiting to acquire commit tokens in TiDB's batch executor. Critical for diagnosing transaction commit bottlenecks and identifying when the commit concurrency limit is causing delays, especially when p99 latency spikes indicate resource contention.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 201, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Statement Lock Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of lock keys acquired per statement in TiDB transactions. Helps diagnose transaction contention issues and identify statements that acquire excessive locks, which can lead to performance degradation and deadlocks.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 67, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction retry attempts in TiDB. Helps diagnose transaction conflicts, write contention issues, and identify workloads experiencing frequent optimistic lock failures that may benefit from pessimistic locking or query optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 134, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "99 Duration", "group": "", "used_for": "Used for monitoring the 99th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation and detecting slow query patterns that affect the majority of database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 132, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "95 Duration", "group": "", "used_for": "Used for monitoring the 95th percentile query execution latency in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation affecting most queries and understanding which SQL types are experiencing elevated latency.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration 80 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 80th percentile query execution latency for each TiDB instance. Helps identify which specific TiDB nodes are experiencing slower query performance and enables comparison of latency distribution across instances in the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 422, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store Slow Score", "group": "Slow Node Statistics", "used_for": "Used for monitoring the slow score of each TiKV store instance, which indicates how slow a store is performing relative to others in the cluster. Essential for identifying underperforming TiKV nodes that may be causing unbalanced load distribution or impacting overall cluster performance.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n store_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 33, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Write KV Num Rate And Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative count of key-value pairs written per transaction in TiDB. Helps identify write-heavy workloads, diagnose transaction size issues, and detect abnormal spikes in write operations that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 193, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution (80th, 95th, and 99th percentiles) of KV transaction operations between TiDB and TiKV. Essential for diagnosing transaction performance issues, identifying slow transaction patterns, and detecting when transaction latency degrades beyond acceptable thresholds.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 130, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "80 Duration", "group": "", "used_for": "Used for monitoring the 80th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Helps identify typical query performance patterns and detect when the majority of queries are experiencing slowdowns, making it useful for capacity planning and performance baseline analysis.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 425, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the count and distribution of Raft snapshot operations in TiKV by type (sending, receiving, applying). Essential for diagnosing region replication issues, identifying snapshot traffic bottlenecks, and troubleshooting slow peer recovery or leader transfer problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 25, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration 99 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 99th percentile query execution latency for each TiDB instance individually. Helps identify which specific TiDB nodes are experiencing performance degradation or handling slower queries compared to others in the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration 95 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 95th percentile query execution latency for each TiDB instance. Helps identify performance variations across instances and pinpoint specific TiDB nodes experiencing elevated query latency issues.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 428, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile snapshot size in TiKV, representing the largest snapshots being generated. Critical for identifying abnormally large snapshots that could cause memory pressure, network congestion, or slow Raft replication during region transfers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 209, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Write KV Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of key-value pairs written per transaction in TiDB. Helps identify transactions with abnormally high write volumes that may cause performance bottlenecks or exceed transaction size limits.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring the throughput of KV transaction operations in TiDB, measured as operations per second across different transaction commands (commit, prewrite, etc.). Essential for tracking transaction workload patterns and identifying sudden changes in transaction processing rates that may indicate application behavior changes or performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 424, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent between TiKV instances. Critical for diagnosing cluster rebalancing activity, replica recovery operations, and identifying potential network or storage bottlenecks during large data transfers.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [1m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 197, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Acquire Pessimistic Locks Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of acquiring pessimistic locks in TiDB transactions at p80, p95, and p99 percentiles. Critical for diagnosing transaction contention issues, lock wait times, and identifying performance bottlenecks in pessimistic transaction workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 218, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Connection Idle Duration", "group": "", "used_for": "Used for monitoring the idle time duration of TiDB client connections at various percentiles (p80, p90, p99), separated by transaction state (in_txn vs non-transactional). Essential for identifying connection pool inefficiencies, detecting long-running idle transactions that may hold locks, and diagnosing application-side connection management issues that could impact cluster resource utilization.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 429, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pairs accessed per snapshot operation in TiKV. Critical for identifying extreme cases where transactions read unusually large numbers of keys, which can indicate inefficient queries or potential performance bottlenecks in snapshot isolation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 210, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction write sizes in bytes across TiDB instances. Helps identify large transactions that may cause performance bottlenecks, memory pressure, or exceed transaction size limits, enabling optimization of write workloads and troubleshooting transaction-related issues.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 94, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Failed Query OPM Detail", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the per-minute increase in query execution errors across TiDB instances. Essential for identifying which specific TiDB servers are experiencing query failures and diagnosing application-level errors, connection issues, or SQL syntax problems.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 34, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes Rate And Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative total of bytes written per transaction in TiDB. Helps identify write-heavy workloads, diagnose transaction size anomalies, and track overall write throughput trends that may impact cluster performance or storage capacity.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 427, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiKV, including sending snapshots and applying snapshots during Raft replication. Critical for diagnosing slow region recovery, replica synchronization delays, and identifying performance bottlenecks when TiKV nodes handle snapshot transfers for data replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 432, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot pending applies", "group": "Snapshot", "used_for": "Used for monitoring the number of Raft snapshots waiting to be applied in TiKV instances. High values indicate snapshot application backlog, which can signal slow disk I/O, resource contention, or issues with region recovery after node failures or scaling operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 194, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Send HeartBeat Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the latency of transaction heartbeat messages sent from TiDB to TiKV to keep long-running transactions alive. Critical for diagnosing issues with large transactions or pessimistic locking scenarios where elevated heartbeat latency (especially at p99) may indicate network problems, TiKV overload, or potential transaction timeout risks.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 430, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot Actions", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot operations in TiKV, including snapshot generation, application, and region cleanup activities. Essential for diagnosing cluster rebalancing issues, slow follower catch-up scenarios, and identifying excessive snapshot traffic that may impact I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "sum(delta(\n tikv_server_snapshot_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration 999 By Instance", "group": "Query Detail", "used_for": "Used for monitoring the 99.9th percentile query execution latency for each TiDB instance individually. Essential for identifying which specific TiDB nodes are experiencing tail latency issues or performance degradation, enabling targeted troubleshooting of slow query problems at the instance level.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 272, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Queries In Multi-Statement", "group": "Query Detail", "used_for": "Used for monitoring the average number of queries contained within multi-statement requests and the total rate of queries in multi-statements. Helps identify applications using batch query patterns and diagnose potential performance issues related to multi-statement execution overhead.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/sum(rate(tidb_server_multi_query_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 68, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Internal SQL OPS", "group": "Query Detail", "used_for": "Used for monitoring the throughput of internal SQL operations executed by TiDB's restricted session (system-level queries). Helps identify overhead from background tasks like statistics collection, auto-analyze, and internal maintenance operations that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 195, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Lifetime Reach Counter", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which TiKV client transactions reach their TTL (Time To Live) lifetime in TiDB instances. Helps identify transaction timeout issues and potential lock contention problems when transactions are held too long before commit or rollback.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 184, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiDB Server Status", "group": "Server", "used_for": "Used for monitoring the health status of TiDB worker instances in the cluster. Shows the count of online (up) and offline (down) TiDB servers, helping quickly identify server availability issues and cluster stability problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} == 1)", "count(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} == 0)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 199, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pessimistic Statement Retry OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of pessimistic transaction statement retries in TiDB. Helps identify transaction conflicts and contention issues that cause statements to retry, which can indicate lock conflicts or write-write conflicts affecting transaction throughput.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 434, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Resolved TS Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU utilization of TiKV's Resolved TS worker threads, which track transaction timestamps for maintaining data consistency in distributed transactions. Essential for diagnosing performance bottlenecks in CDC (Change Data Capture) and follower read scenarios where high CPU usage may indicate timestamp resolution delays or excessive workload.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 83, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Safepoint OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of successful safepoint load operations per second in TiDB's transaction system. Helps diagnose transaction GC (garbage collection) coordination issues and identify instances experiencing problems with safepoint synchronization from PD.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"ok\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 112, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Slow Query", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 90th percentile latency breakdown of slow queries in TiDB, including total processing time, coprocessor execution time, and wait time. Essential for diagnosing performance bottlenecks in slow queries by identifying whether delays occur during query processing, data retrieval from TiKV, or resource waiting.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 437, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts", "group": "Resolved TS", "used_for": "Used for monitoring the maximum time gap between the current time and the resolved timestamp in TiKV. Critical for diagnosing data replication lag and identifying issues with CDC (Change Data Capture) or follower read consistency, where large gaps indicate delayed transaction resolution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring memory consumption of TiDB worker instances, including process resident memory and Go heap allocation. Essential for detecting memory leaks, identifying out-of-memory risks, and optimizing resource allocation in distributed SQL processing workloads.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} + go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}", "go_gc_heap_goal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"worker-tidb\"}", "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"worker-tidb\"}", "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"worker-tidb\"}) by (module, instance)", "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 436, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scan lock Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU usage of TiKV's incremental scan lock workers that support Resolved TS functionality. Essential for diagnosing performance issues in CDC (Change Data Capture) and transaction consistency scenarios where excessive CPU consumption may indicate lock scanning bottlenecks or inefficient transaction processing.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiDB server memory consumption, including process resident memory and Go heap allocation. Essential for diagnosing memory leaks, OOM risks, and optimizing memory configuration when TiDB instances experience high memory pressure or stability issues.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_gc_heap_goal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}) by (module, instance)", "tidb_server_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 224, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Commit P99 Backoff", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile backoff duration during transaction commits in TiDB. Critical for diagnosing transaction conflicts, lock contention, and retry storms that cause commit delays and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 212, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Types Per Second", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate of different transaction commit types per second in TiDB, including standard two-phase commits, async commits, and one-phase commits (1PC). Essential for understanding transaction workload patterns and identifying shifts in commit protocol usage that may impact performance or indicate configuration changes.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 438, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Min Resolved TS Region", "group": "Resolved TS", "used_for": "Used for identifying the region with the minimum resolved timestamp in TiKV, which represents the slowest region in terms of transaction timestamp advancement. Critical for diagnosing stale read performance issues and detecting regions that are blocking the resolved timestamp watermark from advancing across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 439, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Max gap of follower safe-ts", "group": "Resolved TS", "used_for": "Used for monitoring the maximum time gap between follower replicas' safe timestamps in TiKV's Resolved TS mechanism. Critical for diagnosing data consistency issues and replication lag that could affect stale read performance and CDC (Change Data Capture) functionality.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 225, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Max SafeTS Gap", "group": "Transaction", "used_for": "Used for monitoring the maximum gap between SafeTS (safe timestamp) and current time in TiDB's transaction processing. Critical for diagnosing stale read performance issues and identifying delays in timestamp synchronization that could affect transaction consistency and read latency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 188, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiDB worker processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause service disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 226, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "SafeTS Update Conuter", "group": "Transaction", "used_for": "Used for monitoring the rate of SafeTS (Safe Timestamp) updates between TiDB and TiKV stores, broken down by result status and store. Essential for diagnosing transaction coordination issues, stale read problems, and detecting communication failures between TiDB and TiKV that could impact transaction consistency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572003, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB worker instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and correlate service disruptions with specific TiDB worker downtime events.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"})" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 168, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiDB worker instances, showing both actual CPU usage rate and the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation, and determining if TiDB instances need vertical scaling or load redistribution.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 61, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiDB worker instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate connection pooling issues or stuck operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_sched_goroutines_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\"worker-tidb.*\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 231, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Assertion", "group": "Transaction", "used_for": "Used for monitoring the rate of prewrite assertion operations in TiDB transactions, categorized by assertion type. Helps diagnose transaction conflicts and optimistic locking issues that may cause transaction latency or failures.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 443, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Check leader duration", "group": "Resolved TS", "used_for": "Used for monitoring the time spent checking Raft leader status during resolved timestamp advancement in TiKV. Critical for diagnosing resolved TS lag issues caused by slow leader verification, which can impact CDC replication and stale read performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 292, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Runtime GC Rate And GOMEMLIMIT", "group": "Server", "used_for": "Used for monitoring Go garbage collection frequency and memory limit (GOMEMLIMIT) in TiDB worker instances. Helps diagnose memory pressure issues by correlating GC rate spikes with configured memory limits, indicating when the process is approaching memory constraints or experiencing excessive garbage collection overhead.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(go_gc_cycles_total_gc_cycles_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}[2m])", "go_gc_gomemlimit_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 8, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances and tracking connection token usage. Essential for diagnosing connection exhaustion issues, identifying connection leaks, and understanding client connection patterns across the cluster.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 440, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Min Safe TS Follower Region", "group": "Resolved TS", "used_for": "Used for identifying which follower region has the minimum safe timestamp in TiKV's Resolved TS mechanism. Critical for diagnosing stale read issues and detecting regions that are lagging behind in timestamp advancement, which can impact transaction consistency and CDC replication.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Execution States Duration", "group": "Transaction", "used_for": "Used for monitoring the P99, P90, and P80 latency of different transaction execution states in TiDB. Essential for diagnosing transaction performance bottlenecks by identifying which state (e.g., lock waiting, commit, rollback) is causing delays in transaction processing.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 441, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts in region leaders", "group": "Resolved TS", "used_for": "Used for monitoring the maximum time gap between the current timestamp and the resolved timestamp across region leaders in TiKV. Critical for diagnosing data replication lag and identifying regions where transaction consistency guarantees may be delayed, particularly important for CDC (Change Data Capture) and follower read scenarios.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Panic And Critial Error", "group": "Server", "used_for": "Used for monitoring panic events and critical errors occurring in TiDB server instances. Essential for detecting severe system failures and stability issues that require immediate attention, as these errors typically indicate serious problems that could impact cluster availability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 435, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Advance ts Worker CPU", "group": "Resolved TS", "used_for": "Used for monitoring CPU utilization of TiKV's advance timestamp worker threads, which are responsible for advancing the resolved timestamp in distributed transactions. Essential for diagnosing performance bottlenecks in the resolved-ts subsystem that could impact CDC (Change Data Capture) and stale read features.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 165, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Prepare Statement Count", "group": "Server", "used_for": "Used for monitoring the number of prepared statements currently cached in TiDB instances. Helps identify potential memory pressure from excessive prepared statement accumulation and diagnose connection pooling or statement lifecycle management issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 254, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Enter State", "group": "Transaction", "used_for": "Used for monitoring the rate at which transactions enter different states in TiDB. Helps identify transaction lifecycle patterns and diagnose issues with transaction state transitions, such as excessive rollbacks or abnormal state changes that may indicate application or database problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 447, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Lock heap size", "group": "Resolved TS", "used_for": "Used for monitoring the memory consumption of the lock heap in TiKV's Resolved TS mechanism. Helps identify memory pressure issues related to transaction lock tracking and diagnose potential memory leaks in the Resolved TS subsystem.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 255, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Execution States Duration", "group": "Transaction", "used_for": "Used for monitoring the time spent in different transaction execution states (idle, running, committing, etc.) in TiDB. Helps diagnose transaction performance bottlenecks by identifying which state consumes the most time and comparing it against total transaction duration.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_txn_state_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 253, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction With Lock Execution States Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution (p80, p90, p99) of transactions that hold locks across different execution states in TiDB. Critical for diagnosing lock contention issues and identifying when transactions holding locks experience abnormal delays, which can indicate deadlocks, lock wait timeouts, or resource bottlenecks affecting transactional workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 90, 99 ], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", has_lock=\"true\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 446, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Fail advance ts count", "group": "Resolved TS", "used_for": "Used for monitoring failures in advancing the resolved timestamp in TiKV, which is critical for maintaining data consistency in distributed transactions. Helps diagnose issues with stale peer detection and resolved TS progression that can impact CDC (Change Data Capture) and follower read functionality.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 261, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction State Count Change", "group": "Transaction", "used_for": "Used for tracking the rate of change in transaction state transitions within TiDB sessions. Helps identify transaction state management issues, such as transactions getting stuck in certain states or abnormal state transition patterns that could indicate deadlocks or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 111, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Get Token Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of token acquisition in TiDB's connection management system. Critical for diagnosing connection bottlenecks when the server reaches its maximum connection limit or experiences token pool contention.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 444, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CheckLeader request region count $optional_quantile", "group": "Resolved TS", "used_for": "Used for monitoring the distribution of region counts per CheckLeader request in TiKV's Resolved TS mechanism. Helps diagnose performance issues related to leader checking operations and identify when requests are handling unusually large numbers of regions, which could impact transaction consistency guarantees.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 450, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Pending command size", "group": "Resolved TS", "used_for": "Used for monitoring the total size of pending commands in TiKV's Resolved TS channel. Helps diagnose memory pressure and replication lag issues when commands accumulate faster than they can be processed, indicating potential bottlenecks in the Resolved TS subsystem.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 333, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pipelined Flush Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of keys flushed in pipelined write operations during TiDB transactions. Helps diagnose transaction write efficiency and identify potential bottlenecks when large batches of keys are being committed to TiKV.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 82, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Keep Alive OPM", "group": "Server", "used_for": "Used for monitoring the rate of keep-alive operations per minute from TiDB instances to the monitoring system. Helps verify monitoring connectivity health and detect potential network issues or monitoring agent failures when keep-alive signals drop or become irregular.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 211, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Client Data Traffic", "group": "Server", "used_for": "Used for monitoring the network traffic volume between TiDB servers and clients, broken down by traffic type (inbound/outbound). Essential for identifying network bottlenecks, unusual data transfer patterns, and diagnosing connectivity issues or bandwidth saturation problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 205, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Disconnection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total number of client disconnections from TiDB instances, categorized by disconnection result type. Essential for diagnosing connection stability issues, identifying abnormal client disconnects, and troubleshooting network or client-side problems that may impact application connectivity.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 449, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Observe region status", "group": "Resolved TS", "used_for": "Used for monitoring the distribution of region resolution statuses in TiKV's Resolved TS mechanism. Helps diagnose issues with CDC (Change Data Capture) and transaction consistency by tracking which regions are successfully resolving timestamps versus those experiencing delays or failures.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 445, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CheckLeader request size$optional_quantile", "group": "Resolved TS", "used_for": "Used for monitoring the size and item count of CheckLeader requests in TiKV's Resolved TS mechanism at configurable percentiles. Helps diagnose network overhead and batch efficiency issues in the leader checking process, which is critical for maintaining transaction consistency across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 256, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Transaction Leave State", "group": "Transaction", "used_for": "Used for monitoring the rate at which transactions leave different states in TiDB, tracking transaction lifecycle transitions per minute. Helps diagnose transaction processing bottlenecks and identify abnormal transaction state changes that may indicate commit/rollback issues or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 448, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial scan backoff duration", "group": "Resolved TS", "used_for": "Used for monitoring the backoff duration during initial scan operations in TiKV's Resolved TS mechanism. Helps diagnose delays in establishing consistent snapshots for CDC or follower reads, particularly when initial scans encounter conflicts or need to retry.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 334, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pipelined Flush Size", "group": "Transaction", "used_for": "Used for monitoring the distribution of batch sizes in TiDB's pipelined flush operations when committing transactions to TiKV. Helps diagnose transaction commit efficiency and identify whether batching is working effectively to reduce network round-trips during writes.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 191, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Skip Binlog Count", "group": "Server", "used_for": "Used for monitoring the total count of critical errors related to skipped binlog writes in TiDB instances. Essential for detecting binlog replication failures that could lead to data inconsistency between TiDB and downstream systems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 442, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Min Leader Resolved TS Region", "group": "Resolved TS", "used_for": "**Used For:**\n\nUsed for identifying the region with the minimum resolved timestamp among all leader regions in TiKV. Critical for diagnosing Resolved TS lag issues and pinpointing which specific region is blocking transaction consistency advancement, especially when troubleshooting stale read or CDC replication delays.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 166, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Time Jump Back OPS", "group": "Server", "used_for": "Used for monitoring the frequency of system clock backward jumps on TiDB instances. Critical for detecting time synchronization issues that can cause transaction timestamp anomalies, data consistency problems, and TSO (Timestamp Oracle) conflicts in distributed transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 49, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Events OPM", "group": "Server", "used_for": "Used for monitoring the rate of server events occurring in TiDB instances, measured as operations per minute over a 10-minute window. Helps identify unusual event patterns, server state changes, or potential issues by tracking the frequency of internal server events like connection changes, schema updates, or other significant operational activities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 452, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring CPU usage of Point-in-Time Restore (PITR) SST file processing threads in TiKV. Essential for diagnosing performance bottlenecks during backup restoration operations and identifying whether CPU resources are constraining restore speed.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 156, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Parse Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL parsing duration in TiDB. Helps identify performance bottlenecks in the SQL parsing phase, particularly when complex queries or high query volumes cause parsing delays before execution begins.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 296, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Fair Locking Keys", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB's transaction system, categorized by locking type. Helps diagnose transaction contention issues and identify when aggressive locking mechanisms are being triggered due to lock conflicts or high concurrency scenarios.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 171, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "RCCheckTS WriteConflict Num", "group": "Server", "used_for": "Used for monitoring the number of write conflicts detected during read-committed isolation level timestamp checks in TiDB. Helps diagnose transaction contention issues and concurrent write conflicts that may cause transaction retries or failures.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_rc_check_ts_conflict_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 335, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pipelined Flush Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of pipelined flush operations during transaction commits in TiDB, showing p95, p99, and p999 percentiles. Critical for diagnosing transaction commit performance issues and identifying when the pipelined write optimization is experiencing delays that could impact overall transaction throughput.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_pipelined_flush_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))/ sum(rate(tidb_tikvclient_pipelined_flush_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 453, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "P99 RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the 99th percentile latency of Point-in-Time Restore (PITR) related RPC operations in TiKV, including import RPC calls, apply operations, and engine requests. Critical for diagnosing performance bottlenecks during backup restoration and identifying slow PITR operations that may impact recovery time objectives.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 455, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import RPC Count", "group": "Point In Time Restore", "used_for": "Used for monitoring the count of import RPC apply operations in TiKV during Point in Time Restore (PITR). Helps track restore progress and identify potential bottlenecks or failures in the data import process when recovering to a specific point in time.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 167, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Handshake Error OPS", "group": "Server", "used_for": "Used for monitoring the rate of client connection handshake failures per TiDB instance. Essential for diagnosing authentication issues, SSL/TLS configuration problems, or client compatibility errors that prevent successful database connections.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 169, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Execution Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile execution duration of SQL statements in TiDB. Essential for identifying slow query execution issues and understanding typical query performance under load, helping diagnose when queries are taking longer than expected to complete.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572020, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Active Users", "group": "Server", "used_for": "Used for monitoring the number of currently active user connections to the TiDB worker cluster. Helps identify connection pool usage, detect connection leaks, and assess whether the cluster is approaching connection limits during peak loads.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_active_users{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572016, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Internal Sessions", "group": "Server", "used_for": "Used for monitoring the number of internal sessions in TiDB worker nodes, which are system-level connections used for background tasks like statistics collection, auto-analyze, and internal queries. Helps identify resource contention or bottlenecks caused by excessive internal operations competing with user queries.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_internal_sessions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 69, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring transaction operations per second in TiDB, broken down by transaction type (commit/rollback) and mode (optimistic/pessimistic), with separate tracking for general user transactions and internal system transactions. Essential for understanding transaction throughput patterns and identifying anomalies in transaction processing rates.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (type, txn_mode)", "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"internal\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 415, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages sent from TiKV to PD and the number of pending heartbeats. Critical for diagnosing communication issues between TiKV and PD, detecting network problems, or identifying when TiKV instances are struggling to maintain cluster coordination.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 458, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read File into Memory Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the time taken to read backup files into memory during Point-in-Time Restore (PITR) operations in TiKV. Critical for diagnosing slow restore performance and identifying bottlenecks in the file download phase of backup recovery.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 154, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Compile Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL compilation latency in TiDB, measuring the time spent parsing and optimizing SQL statements before execution. Critical for diagnosing slow query issues caused by complex query plans or inefficient SQL compilation, especially when queries are slow despite fast execution times.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 454, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of import RPC operations during Point In Time Restore (PITR) operations in TiKV. Helps diagnose restore performance issues by tracking the frequency of apply and other import-related RPC requests, excluding mode switches.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 258, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Plan Cache Miss OPS", "group": "Executor", "used_for": "Used for monitoring the rate of plan cache misses per second in TiDB, categorized by miss type. Essential for diagnosing execution plan cache efficiency issues and identifying why prepared statements aren't reusing cached plans, which can impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 460, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Request Throughput", "group": "Point In Time Restore", "used_for": "Used for monitoring the throughput of apply requests during Point In Time Restore (PITR) operations in TiKV. Essential for tracking restore performance and identifying bottlenecks when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 461, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Downloaded File Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the total size of files downloaded during Point-in-Time Restore (PITR) operations in TiKV. Helps track restore progress and diagnose slow data transfer issues when recovering from backups.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 456, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Cache Events", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of cache events during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose cache efficiency and performance issues during backup restoration, particularly when tracking how frequently the apply cache is being accessed or modified.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 91, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache feature, broken down by cache operation type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Expensive Executors OPS", "group": "Executor", "used_for": "Used for monitoring the rate of expensive executor operations in TiDB, categorized by executor type. Helps identify query patterns that consume excessive resources (memory, CPU) and may impact cluster performance, enabling quick detection of inefficient query execution plans.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 249, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Read From Table Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of read operations served by TiDB's table cache feature. Helps evaluate table cache effectiveness and identify whether frequently accessed small tables are being efficiently cached in memory to reduce storage layer access.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_read_from_tablecache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 457, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Overall RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the overall RPC request duration in TiKV's Point In Time Restore (PITR) operations, specifically tracking apply requests during backup/restore processes. Essential for diagnosing performance bottlenecks and latency issues when restoring data to a specific point in time.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Statement Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of statement counts per transaction in TiDB. Helps identify transactions with excessive statements that may cause performance issues or indicate inefficient application logic requiring optimization.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 295, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Fair Locking Usage", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB transactions, categorized by lock type. Helps diagnose transaction contention issues and understand locking behavior patterns when multiple transactions compete for the same resources.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_session_transaction_fair_locking_usage{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 72, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile transaction duration in TiDB, broken down by transaction type and mode. Essential for identifying transaction latency issues, comparing pessimistic vs. optimistic transaction performance, and diagnosing slow commit problems that impact application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, txn_mode))", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 67, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Retry Num", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of transaction retry attempts in TiDB. Helps diagnose transaction conflicts, write contention issues, and identify workloads experiencing frequent optimistic lock failures that may benefit from pessimistic locking or query optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 459, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Queuing Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time backup/restore requests spend waiting in queue before being processed by TiKV's import engine during Point-in-Time Restore operations. Critical for identifying bottlenecks in restore performance and determining if the system is overloaded with concurrent restore requests.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 269, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Plan Cache Memory Usage", "group": "Executor", "used_for": "Used for monitoring the memory consumption of TiDB's plan cache on each instance. Essential for diagnosing memory pressure issues and determining if plan cache memory usage is contributing to OOM problems or needs tuning.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 271, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Plan Cache Plan Num", "group": "Executor", "used_for": "Used for monitoring the total number of cached execution plans stored in each TiDB instance's plan cache. Helps assess plan cache memory usage and effectiveness, useful when diagnosing performance issues related to plan cache capacity or investigating why queries aren't benefiting from cached plans.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_plan_num_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763574010, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "IndexLookUp Duration", "group": "Executor", "used_for": "Used for monitoring the execution latency of IndexLookUp operations in TiDB at p80, p95, and p99 percentiles. Critical for diagnosing performance issues with queries that use index lookups to retrieve table data, helping identify when secondary index access becomes a bottleneck.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_executor_index_lookup_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"enable_index_lookup_push_down\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_executor_index_lookup_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"enable_index_lookup_push_down\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_executor_index_lookup_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"enable_index_lookup_push_down\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 463, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Blocked by Concurrency Time", "group": "Point In Time Restore", "used_for": "Used for monitoring time spent waiting for concurrency permits during Point-in-Time Restore (PITR) operations in TiKV. Helps identify bottlenecks when restore tasks are queued due to concurrency limits, indicating whether increasing parallelism could improve restore performance.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 464, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Request Speed", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of Point-in-Time Restore (PITR) apply requests being processed by TiKV instances. Helps track restore operation throughput and identify performance bottlenecks during cluster recovery from backup snapshots.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 465, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Cached File in Memory", "group": "Point In Time Restore", "used_for": "Used for monitoring the amount of cached file data held in memory during Point-in-Time Restore (PITR) operations on each TiKV instance. Helps identify memory pressure and optimize cache usage when restoring data from backups.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Session Retry Error OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of transaction retry errors in TiDB sessions, categorized by error type and SQL type. Essential for diagnosing transaction conflicts, lock timeouts, and other retry-related issues that impact transaction success rates and application performance.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, sql_type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 4, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring the throughput of KV transaction operations in TiDB, broken down by operation type (e.g., commit, prewrite, get). Essential for identifying transaction workload patterns, detecting sudden changes in transaction volume, and diagnosing performance bottlenecks in the transaction processing pipeline.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 462, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Batch Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the batch size distribution of data apply operations during Point-in-Time Restore (PITR) in TiKV. Helps assess restore performance and identify whether data is being applied in optimal batch sizes, which is critical for diagnosing slow PITR operations.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 470, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU consumption by TiKV backup worker threads and I/O operations during backup tasks. Essential for identifying backup performance bottlenecks, detecting resource contention during backup operations, and ensuring backups don't excessively impact cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 466, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Engine Requests Unfinished", "group": "Point In Time Restore", "used_for": "Used for monitoring the number of unfinished engine requests during Point In Time Restore (PITR) operations in TiKV. Helps track restore progress and identify potential bottlenecks or stalled import operations that may delay recovery completion.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763574013, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "IndexLookUp Rows", "group": "Executor", "used_for": "Used for monitoring the rate of rows processed by IndexLookUp operations in TiDB's executor, including both normal index lookups and push-down optimizations. Helps identify inefficient table access patterns and evaluate the effectiveness of index lookup push-down optimizations for query performance tuning.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_index_lookup_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) ", "sum(rate(tidb_executor_index_lookup_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"normal\"}[2m])) by (type)", "sum(rate(tidb_executor_index_lookup_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"index_lookup_push_down_.*\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572017, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Plan Cache Process Duration", "group": "Executor", "used_for": "Used for monitoring the time spent processing plan cache operations in TiDB, including cache hits, misses, and evictions. Essential for diagnosing query performance issues related to plan cache efficiency and identifying when cache misses are causing increased query latency.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_process_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", sql_type!=\"internal\"}[2m])) by (le, type) / sum(rate(tidb_server_plan_cache_process_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", sql_type!=\"internal\"}[2m])) by (le, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 33, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Write KV Num Rate And Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative count of key-value pairs written per transaction in TiDB. Helps identify transaction write patterns, detect abnormal write amplification, and diagnose issues where transactions are writing unexpectedly large numbers of keys, which can impact cluster performance and resource consumption.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 193, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution of KV transaction operations in TiDB at the 80th, 95th, and 99th percentiles. Essential for identifying transaction performance degradation, diagnosing slow transaction types (commit, prewrite, etc.), and detecting tail latency issues that impact user experience.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 467, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Apply Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time spent applying restored data during Point-in-Time Recovery (PITR) operations in TiKV. Critical for tracking restore performance and identifying bottlenecks when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763574012, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "IndexLookUp Row Num With PushDown Enabled", "group": "Executor", "used_for": "Used for monitoring the number of rows processed by IndexLookUp operations when pushdown optimization is enabled in TiDB. Helps evaluate the effectiveness of index pushdown optimization and identify cases where large row counts may indicate inefficient query patterns or missing index optimizations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_executor_index_lookup_row_number_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"enable_index_lookup_push_down\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 471, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of threads allocated to the backup thread pool on each TiKV instance. Helps diagnose backup performance issues and resource allocation problems when backup operations are slow or consuming excessive resources.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 209, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Write KV Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of key-value pairs written per transaction in TiDB. Helps identify transactions with abnormally high write volumes that may cause performance bottlenecks or exceed transaction size limits.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 468, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Store Memory Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring memory consumption of TiKV's Raft Store component, which handles consensus protocol operations and log replication. Essential for diagnosing memory pressure issues in the Raft layer, especially during Point-in-Time Restore operations or when investigating out-of-memory problems in TiKV instances.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 210, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Write Size Bytes", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction write sizes in bytes across TiDB instances. Helps identify large transactions that may cause performance bottlenecks, memory pressure, or exceed transaction size limits, enabling optimization of write workloads.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 472, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Errors", "group": "Backup & Import", "used_for": "Used for tracking backup operation failures in TiKV instances. Essential for monitoring backup job health and quickly identifying when backup processes encounter errors that could compromise data protection and disaster recovery capabilities.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, error) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 34, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Write Size Bytes Rate And Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative total of bytes written per transaction in TiDB. Helps identify write-heavy workloads, diagnose transaction throughput issues, and detect abnormal spikes in transaction write volume that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 474, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in the default column family during TiKV backup operations. Helps identify backup performance issues and optimize backup chunk sizes by tracking whether SST files are appropriately sized for efficient backup processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Distsql Duration", "group": "Distsql", "used_for": "Used for monitoring the latency of distributed SQL (DistSQL) query execution in TiDB at various percentiles (p50, p90, p99, p999). Essential for diagnosing performance issues in queries that involve distributed data processing across TiKV nodes, helping identify slow distributed operations and tail latency problems in cross-node query execution.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 44, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Regions Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of region counts involved in transaction prewrite operations. Helps identify transactions that span many regions, which can indicate inefficient data access patterns or potential performance bottlenecks in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 312, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Mpp Coordinator Latency", "group": "Executor", "used_for": "Used for monitoring the latency of MPP (Massively Parallel Processing) coordinators in TiDB at different percentiles (p50, p75, p95). Essential for diagnosing performance issues in analytical queries that use TiFlash's MPP execution engine, helping identify coordination overhead and bottlenecks in distributed query processing.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 75, 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.75, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Scan Keys Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per distributed SQL query at different percentiles (p50, p90, p100). Essential for identifying inefficient queries with excessive key scans, which can indicate missing indexes, full table scans, or poorly optimized query patterns that impact cluster performance.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Send HeartBeat Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of transaction heartbeat operations sent from TiDB to TiKV at p80, p95, and p99 percentiles. Critical for diagnosing issues with long-running transactions that require periodic heartbeats to prevent lock timeouts, especially when investigating transaction commit failures or lock wait problems.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 60, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Distsql Partial QPS", "group": "Distsql", "used_for": "Used for monitoring the rate of partial distributed SQL scan operations per second in TiDB. Helps identify when queries are performing partial scans (scanning subset of keys), which is useful for diagnosing query execution patterns and potential performance issues related to distributed query processing.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 195, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Lifetime Reach Counter", "group": "Transaction", "used_for": "Used for monitoring the rate at which TiKV client transactions reach their TTL (Time To Live) expiration in TiDB. Helps identify transaction timeout issues and potential lock contention problems when transactions are held too long before commit.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 197, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Acquire Pessimistic Locks Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of acquiring pessimistic locks in TiDB transactions at p80, p95, and p99 percentiles. Critical for diagnosing transaction performance issues, lock contention problems, and identifying when pessimistic locking operations are causing transaction delays or timeouts.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 473, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST files in the Write column family during TiKV backup operations. Helps diagnose backup performance issues and identify abnormally large SST files that could impact backup efficiency and storage consumption.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763574009, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "IndexLookUp OPS", "group": "Executor", "used_for": "Used for monitoring the throughput of IndexLookUp operations in TiDB, tracking the rate of expensive IndexLookUp executions, row lookups, and coprocessor tasks. Essential for identifying performance bottlenecks when queries use index lookups to retrieve table data, helping diagnose inefficient query patterns and index usage issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"IndexLookUpExecutor\"}[2m])) ", "sum(rate(tidb_executor_index_lookup_row_number_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"enable_index_lookup_push_down\"}[2m]))", "sum(rate(tidb_executor_index_lookup_cop_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"index_scan_.*\"}[2m]))", "sum(rate(tidb_executor_index_lookup_cop_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"index_scan_with_lookup_push_down\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 311, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Mpp Coordinator Counter", "group": "Executor", "used_for": "Used for monitoring the rate of MPP (Massively Parallel Processing) coordinator operations in TiDB, categorized by operation type. Essential for tracking the activity and health of MPP query coordination, helping diagnose issues with distributed analytical query execution and identifying bottlenecks in the MPP framework.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_mpp_coordinator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 83, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Safepoint OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of successful GC safepoint load operations per second in TiDB. Helps diagnose transaction GC coordination issues and identify instances experiencing problems retrieving the cluster's safe timestamp for MVCC cleanup.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ok\"}[2m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 475, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file generation during TiKV backup operations, measured in bytes per second. Essential for evaluating backup performance, identifying bottlenecks in backup processes, and ensuring backup operations meet expected data transfer rates.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, cf) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Scan Keys Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of keys scanned in partial distributed SQL requests at p80, p90, and p100 percentiles. Helps diagnose inefficient query execution patterns where excessive key scanning indicates missing indexes, poor query optimization, or table scan issues causing high query latency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 476, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time spent scanning SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup scan phase that may cause slow or failed backups.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 175, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Coprocessor Cache", "group": "Distsql", "used_for": "Used for monitoring the hit rate and effectiveness of TiDB's coprocessor cache, which stores results from TiKV coprocessor requests. Helps identify cache efficiency and determine if increasing cache size could reduce redundant queries to TiKV, improving query performance.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 59, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of partial results returned per distributed SQL query at different percentiles (p50, p90, max). Helps diagnose query execution efficiency and identify queries that require excessive partial result merging, which can indicate suboptimal query plans or data distribution issues.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 226, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "SafeTS Update Conuter", "group": "Transaction", "used_for": "Used for monitoring the rate of SafeTS (Safe Timestamp) updates in TiDB's transaction system, broken down by result status and TiKV store. Essential for diagnosing transaction coordination issues and identifying stores with SafeTS synchronization problems that could impact transaction consistency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, store)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 478, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time taken to save SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup process when backup jobs are running slower than expected.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 196, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Commit Token Wait Duration", "group": "Transaction", "used_for": "Used for monitoring the wait time transactions spend acquiring commit tokens in TiDB's batch executor at p80, p95, and p99 percentiles. Critical for diagnosing transaction commit bottlenecks caused by token contention when concurrent commits exceed available token capacity, which directly impacts transaction latency during high-concurrency workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 225, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Max SafeTS Gap", "group": "Transaction", "used_for": "Used for monitoring the maximum gap between SafeTS (safe timestamp) and current time in TiDB's transaction system. Critical for diagnosing stale read performance issues and identifying delays in timestamp synchronization that could affect transaction consistency and read latency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 479, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the 99.9th percentile latency of SST file backup operations in TiKV. Critical for diagnosing backup performance issues and identifying slow backup ranges that could impact backup completion times or cluster stability during backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 212, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Types Per Second", "group": "Transaction", "used_for": "Used for monitoring the rate of different transaction commit types in TiDB, including standard two-phase commit, async commit, and one-phase commit (1PC) transactions. Essential for understanding transaction workload patterns and identifying which commit protocols are being used, helping diagnose transaction performance issues and validate optimization strategies like async commit adoption.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 483, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IO Utilization", "group": "Backup & Import", "used_for": "Used for monitoring disk I/O utilization percentage on TiKV nodes during backup and import operations. Helps identify I/O bottlenecks and resource contention that may impact backup/restore performance or cause latency spikes during data import tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, device) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 477, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time spent scanning SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup scan phase that may indicate storage I/O problems or excessive data volume.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 480, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the time taken to create connections or initialize external storage systems (like S3, GCS) during backup and restore operations in TiKV. Critical for diagnosing slow backup/restore startup issues and external storage connectivity problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 231, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Assertion", "group": "Transaction", "used_for": "Used for monitoring the rate of transaction prewrite assertions in TiDB, categorized by assertion type. Helps diagnose transaction conflicts and optimistic locking issues that may cause transaction retries or failures.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 199, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pessimistic Statement Retry OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of pessimistic transaction statement retries in TiDB. Helps identify transaction conflicts and contention issues that cause statements to retry, which can indicate lock conflicts or write-write conflicts in pessimistic transaction mode.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 14, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Distsql QPS", "group": "Distsql", "used_for": "Used for monitoring the queries per second of distributed SQL (DistSQL) requests in TiDB, broken down by coprocessor type (table scan, index scan, etc.). Essential for understanding the volume and distribution of pushdown queries to TiKV, helping diagnose performance issues related to distributed query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (copr_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 481, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of creating connections to external storage systems (S3, GCS, etc.) during backup and restore operations in TiKV. Critical for diagnosing slow backup/restore performance caused by external storage connection issues, with p99 and p9999 percentiles helping identify tail latency problems in storage access.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 224, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Commit P99 Backoff", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile backoff duration during transaction commits in TiDB. Critical for diagnosing transaction commit delays caused by lock conflicts, region unavailability, or other retry scenarios that force TiDB to back off before retrying commit operations.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 11, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiClient Region Error OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of region-related errors encountered by TiDB clients when accessing TiKV, categorized by error type (e.g., not_leader, epoch_not_match, server_is_busy). Critical for diagnosing region availability issues, leadership changes, and cluster instability that impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 223, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Replica Selector Failure Per Second", "group": "KV Errors", "used_for": "Used for monitoring the rate of replica selector failures in TiDB's TiKV client, categorized by failure type. Critical for diagnosing issues with TiDB's ability to select appropriate TiKV replicas for read operations, which can indicate network problems, region unavailability, or follower read configuration issues.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 32, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Lock Resolve OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of lock resolution operations in TiDB's transaction processing system. Essential for diagnosing transaction conflicts and identifying when the cluster is experiencing high contention due to locks that need to be resolved before transactions can proceed.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Backoff Duration", "group": "KV Errors", "used_for": "Used for monitoring the time TiDB spends in backoff retry mechanisms when encountering KV-layer errors (like region unavailable, lock conflicts, or server busy). Critical for diagnosing transaction conflicts, region scheduling issues, and TiKV overload situations that force TiDB to repeatedly retry operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 484, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import operations. Essential for diagnosing performance bottlenecks and resource contention when importing large datasets or restoring backups.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 53, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Backoff OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of TiKV client backoff operations in TiDB, categorized by backoff type. Essential for identifying when TiDB encounters resource contention, region unavailability, or lock conflicts that force it to retry operations, helping diagnose performance degradation and cluster health issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Coprocessor Seconds 999", "group": "Distsql", "used_for": "Used for monitoring the 99.9th percentile latency of coprocessor requests from TiDB to TiKV. Critical for identifying tail latency issues in distributed query execution, particularly when complex queries involving table scans or aggregations experience performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", store!=\"0\", scope=\"false\", type=\"Cop\"}[2m])) by (le,instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 253, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction With Lock Execution States Duration", "group": "Transaction", "used_for": "Used for monitoring the duration of transaction execution states that hold locks, at p80, p90, and p99 percentiles. Critical for diagnosing lock contention issues, identifying transactions holding locks for extended periods, and troubleshooting transaction blocking or deadlock scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 172, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Request OPS", "group": "KV Request", "used_for": "Used for monitoring the operations per second of KV requests sent from TiDB to TiKV, broken down by request type and instance. Essential for identifying which types of KV operations (Get, Scan, Prewrite, Commit, etc.) are consuming the most throughput and diagnosing performance bottlenecks in the TiDB-TiKV communication layer.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=\"false\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 255, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Execution States Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the time spent in different transaction execution states (idle, running, committing) and overall transaction duration in TiDB. Essential for diagnosing transaction performance bottlenecks, identifying whether delays occur during execution or commit phases, and understanding transaction lifecycle patterns.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_txn_state_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 252, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Execution States Duration", "group": "Transaction", "used_for": "Used for monitoring the P99, P90, and P80 latency of different transaction execution states in TiDB. Essential for diagnosing transaction performance bottlenecks by identifying which state (e.g., lock waiting, commit, rollback) is causing delays in transaction processing.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 48, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Request Duration 99 By Store", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of KV requests from TiDB to individual TiKV stores. Essential for identifying which specific TiKV stores are experiencing slow response times and diagnosing store-level performance bottlenecks or hotspots.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le, store))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 485, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of active SST import threads in TiKV during backup and restore operations. Helps diagnose import performance issues and verify that sufficient threads are allocated for data ingestion tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 486, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Errors", "group": "Backup & Import", "used_for": "Used for tracking the rate of errors occurring during data import operations in TiKV. Essential for diagnosing import failures and monitoring the health of backup restoration or bulk data loading processes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, error, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 256, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Leave State", "group": "Transaction", "used_for": "Used for monitoring the rate at which transactions leave different states in TiDB. Helps identify transaction lifecycle bottlenecks and abnormal state transitions that may indicate commit/rollback issues or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 261, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction State Count Change", "group": "Transaction", "used_for": "Used for tracking the rate of change in transaction state transitions within TiDB sessions. Helps identify transaction state management issues, such as transactions getting stuck in certain states or abnormal state transition patterns that could indicate deadlocks or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 487, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of import RPC operations in TiKV, tracking both p99 and p9999 percentiles. Critical for diagnosing performance issues during data import operations such as Lightning imports or backup restoration, helping identify slow import requests and bottlenecks in the import pipeline.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 489, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import RPC Count", "group": "Backup & Import", "used_for": "Used for monitoring the count of import RPC requests in TiKV during data import operations. Helps track import activity volume and identify potential bottlenecks or failures in backup restoration and bulk data loading processes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 488, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Backup & Import", "used_for": "Used for monitoring the rate of import RPC operations in TiKV, excluding mode switches. Essential for tracking data import activity and diagnosing performance issues during bulk data loading operations like BR restore or Lightning imports.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 295, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Fair Locking Usage", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB transactions, categorized by lock type. Helps diagnose transaction contention issues and identify whether pessimistic locking is being used efficiently to handle concurrent transaction conflicts.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_session_transaction_fair_locking_usage{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 250, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Region Cache Error OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of region cache errors in TiDB's TiKV client, broken down by error type. Critical for diagnosing region metadata issues, stale cache problems, and network connectivity failures that can cause query retries and performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"err\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 492, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Read SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST file read operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data or migrating large datasets into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"read\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 220, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS By Type", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded by TiDB to other TiKV nodes, broken down by request type and result status. Essential for diagnosing cross-region request patterns, identifying forwarding bottlenecks, and troubleshooting issues related to stale read or follower read configurations.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 490, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of import-related RPC operations in TiKV, specifically write and download requests during data import processes. Essential for diagnosing performance bottlenecks during backup restoration or bulk data import operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Request Duration 99 By Type", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of different KV request types (Get, Scan, Prewrite, Commit, etc.) sent from TiDB to TiKV. Essential for identifying which specific request types are experiencing performance degradation and diagnosing transaction or query bottlenecks at the storage layer.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le,type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 334, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pipelined Flush Size", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of batch sizes in TiDB's pipelined transaction flush operations to TiKV. Helps diagnose transaction commit efficiency and identify whether transactions are being batched optimally, which is critical for understanding write throughput and potential I/O bottlenecks.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 296, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Fair Locking Keys", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB's transaction system, categorized by lock type. Helps diagnose transaction contention issues and identify when aggressive locking mechanisms are being triggered due to lock conflicts or high concurrency scenarios.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 219, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests being forwarded between TiKV stores in operations per second. Essential for diagnosing load balancing issues, identifying hot regions being redirected, and detecting potential network or store availability problems that trigger request forwarding.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (from_store, to_store, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 333, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pipelined Flush Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of keys flushed in TiDB's pipelined transaction commit process. Helps diagnose transaction commit efficiency and identify potential bottlenecks when large batches of keys are being written, which can impact overall transaction throughput and latency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tidb_tikvclient_pipelined_flush_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 164, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Region Cache OK OPS", "group": "KV Request", "used_for": "Used for monitoring the successful operations per second on TiDB's region cache, broken down by operation type (e.g., hits, misses, updates). Essential for evaluating region cache effectiveness and diagnosing KV request routing issues that may impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"ok\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 254, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Transaction Enter State", "group": "Transaction", "used_for": "Used for monitoring the rate at which transactions enter different states in TiDB. Helps identify transaction lifecycle patterns and diagnose issues with transaction state transitions, such as excessive rollbacks or abnormal state changes that may indicate application or database problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 482, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of checksum requests processed by TiKV's coprocessor during backup and import operations. Critical for diagnosing performance bottlenecks in data verification tasks, especially when backup jobs are slow or experiencing tail latency issues at p99 and p9999 percentiles.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 491, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Wait Duration", "group": "Backup & Import", "used_for": "Used for monitoring the queue wait time during data import operations in TiKV. Helps diagnose import performance bottlenecks by identifying delays before import tasks begin processing, which is critical when troubleshooting slow bulk data loading or backup restoration operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9999, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Successful KV Request Wait Duration", "group": "KV Request", "used_for": "Used for monitoring the P99 and P90 wait duration of successful KV requests in TiDB's resource manager client. Essential for diagnosing resource control bottlenecks and identifying when requests are being throttled or delayed due to resource group quota limitations.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(resource_manager_client_request_success_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (instance, resource_group, le))", "histogram_quantile(0.9, sum(rate(resource_manager_client_request_success_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (instance, resource_group, le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 431, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Snapshot transport speed", "group": "Snapshot", "used_for": "Used for monitoring the rate of snapshot data transport and generation in TiKV, measured in bytes per second. Critical for diagnosing snapshot replication performance issues, identifying bandwidth bottlenecks during region transfers, and understanding the speed of data synchronization between TiKV nodes during scaling or recovery operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 156, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Parse Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL parsing duration in TiDB, broken down by SQL statement type. Essential for identifying parsing performance bottlenecks and detecting complex SQL statements that take excessive time to parse before execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 263, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "RPC Layer Latency", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile network latency between TiDB and TiKV at the RPC layer. Critical for diagnosing network-related performance issues and distinguishing network overhead from actual request processing time in KV operations.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store))", "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 494, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of import ingest RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when ingesting SST files during backup restoration or bulk data import operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 493, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks in backup restoration or bulk data import processes, especially when imported data needs to be rewritten to match the cluster's key ranges.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 262, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Region Duration", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of loading region information into TiDB's region cache from PD. Critical for diagnosing slow KV request issues caused by region cache misses or PD communication delays, especially when queries experience unexpected latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 306, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stale Read Hit/Miss OPS", "group": "KV Request", "used_for": "Used for monitoring the throughput of stale read requests in TiDB, showing the rate of cache hits versus misses per second. Helps evaluate the effectiveness of stale read optimization and identify whether queries are successfully utilizing cached data to reduce latency.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 169, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Execution Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile execution duration of SQL statements in TiDB, broken down by SQL type (SELECT, INSERT, UPDATE, etc.). Essential for identifying slow query types and diagnosing execution performance bottlenecks that affect most users.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 335, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pipelined Flush Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the latency of pipelined flush operations during transaction commits in TiDB, showing p95, p99, and p999 percentiles. Critical for diagnosing transaction commit performance issues and identifying when the asynchronous flush mechanism experiences delays that could impact overall transaction throughput.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_tikvclient_pipelined_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_pipelined_flush_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/ sum(rate(tidb_tikvclient_pipelined_flush_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 307, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stale Read Req OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests per second in TiDB, broken down by request type. Essential for tracking the usage and throughput of stale read operations, which allow reading slightly outdated data for improved performance and reduced latency in scenarios where strict consistency is not required.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_req_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 154, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Compile Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL compilation latency in TiDB, broken down by SQL statement type. Critical for diagnosing query plan generation performance issues and identifying SQL types that experience slow compilation times, which can impact overall query execution performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 495, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks during bulk data imports, backup restoration, or Lightning import operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 497, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the download throughput of SST (Sorted String Table) files during data import operations in TiKV. Essential for diagnosing slow import performance and identifying network or I/O bottlenecks when restoring backups or importing large datasets.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 496, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "Backup & Import", "used_for": "Used for monitoring the volume of SST (Sorted String Table) file bytes ingested during TiDB Lightning or BR (Backup & Restore) import operations. Essential for tracking import progress, diagnosing slow data ingestion rates, and identifying potential bottlenecks in the bulk data loading process.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 498, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Local Write keys", "group": "Backup & Import", "used_for": "Used for monitoring the number of keys written during TiDB Lightning's local import process in TiKV. Helps track import progress and diagnose performance issues when bulk loading data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 76, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Expensive Executors OPS", "group": "Executor", "used_for": "Used for tracking the rate of expensive executor operations in TiDB, categorized by executor type. Helps identify query patterns that consume excessive resources (memory, CPU) and may impact cluster performance or cause OOM issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 258, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Plan Cache Miss OPS", "group": "Executor", "used_for": "Used for monitoring the rate of plan cache misses per second in TiDB, categorized by miss type. Essential for diagnosing execution plan cache efficiency issues and identifying why queries aren't reusing cached plans, which can impact query performance and increase CPU overhead.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 337, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Client-side Slow Score", "group": "KV Request", "used_for": "Used for monitoring the slow score of TiKV stores as perceived by TiDB clients, indicating store responsiveness and health. Essential for identifying problematic TiKV nodes that may be causing request slowdowns or timeouts, helping diagnose unbalanced load or hardware issues affecting specific stores.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_store_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 249, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Read From Table Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of read operations served by TiDB's table cache feature. Helps assess table cache effectiveness and identify whether cached tables are being actively read, which is crucial for validating cache configuration and optimizing small, frequently-accessed table performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_read_from_tablecache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 500, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL Expired", "group": "Backup & Import", "used_for": "Used for monitoring the count of expired Time-To-Live (TTL) entries in TiKV's raw backup data. Helps identify potential data retention issues and assess the effectiveness of TTL-based data cleanup mechanisms during backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD Client CMD OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second (OPS) of PD client commands from TiDB instances, excluding TSO requests. Helps identify communication patterns between TiDB and PD, and diagnose issues with metadata operations like region routing, schema changes, or cluster configuration queries.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type!=\"tso\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 271, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Plan Cache Plan Num", "group": "Executor", "used_for": "Used for monitoring the total number of cached execution plans in TiDB's plan cache per instance. Helps assess plan cache utilization and identify potential memory pressure from excessive cached plans or inefficient cache usage patterns.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_plan_num_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 269, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Plan Cache Memory Usage", "group": "Executor", "used_for": "Used for monitoring the memory consumption of TiDB's plan cache on each instance. Essential for diagnosing memory pressure issues and determining if plan cache memory usage is contributing to OOM problems or needs tuning.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 501, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "cloud request", "group": "Backup & Import", "used_for": "Used for monitoring the rate of cloud storage requests (such as S3 or GCS operations) from TiKV instances during backup and restore operations. Helps diagnose cloud API throttling, network issues, or performance bottlenecks when backing up data to or restoring from cloud storage.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (cloud, req) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 499, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import Local Write bytes", "group": "Backup & Import", "used_for": "Used for monitoring the write throughput (bytes per second) during local data import operations in TiKV. Essential for tracking import performance and diagnosing slow data ingestion issues during backup restoration or bulk data loading scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 91, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache, broken down by cache hit/miss type. Helps evaluate plan cache effectiveness and identify opportunities to improve query performance by increasing cache hit rates.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 35, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD Client CMD Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the latency of PD client commands executed by TiDB instances at p90, p99, and p999 percentiles. Essential for diagnosing communication delays between TiDB and PD, helping identify issues with metadata operations, TSO allocation, or region routing that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 338, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiKV-side Slow Score", "group": "KV Request", "used_for": "Used for monitoring the slow score feedback from each TiKV store as perceived by TiDB instances. Helps identify which TiKV stores are experiencing performance degradation or responding slowly to requests, enabling quick isolation of problematic storage nodes.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_feedback_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 79, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD TSO OPS", "group": "PD Client", "used_for": "Used for monitoring the throughput of TSO (Timestamp Oracle) requests from TiDB to PD, measured in operations per second. Essential for identifying TSO request bottlenecks, validating transaction workload patterns, and diagnosing issues when applications experience timestamp allocation delays or transaction conflicts.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m]))", "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 505, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Advancer Owner", "group": "Backup Log", "used_for": "Used for identifying which TiKV instance is currently serving as the log backup advancer owner. Essential for troubleshooting log backup coordination issues and verifying proper leadership election in the backup process.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "tidb_log_backup_advancer_owner > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 506, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Average Flush Size", "group": "Backup Log", "used_for": "Used for monitoring the average size of backup log files being flushed to storage in TiKV. Helps identify backup efficiency issues and optimize flush batch sizes to balance between backup performance and storage overhead.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572017, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Plan Cache Process Duration", "group": "Executor", "used_for": "Used for monitoring the latency of plan cache operations in TiDB, including cache hits, misses, and evictions. Essential for diagnosing query performance issues related to execution plan caching and identifying when plan cache inefficiency is causing query slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_process_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", sql_type!=\"internal\"}[2m])) by (le, type) / sum(rate(tidb_server_plan_cache_process_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", sql_type!=\"internal\"}[2m])) by (le, type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 504, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Task Status", "group": "Backup Log", "used_for": "Used for monitoring the operational status of backup log tasks on each TiKV instance. Helps quickly identify instances where log backup tasks are paused, failed, or not running properly, enabling rapid troubleshooting of backup pipeline issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 512, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Handle Event Rate", "group": "Backup Log", "used_for": "Used for monitoring the rate at which TiKV processes backup log events during continuous data backup operations. Essential for tracking backup throughput and identifying performance bottlenecks or slowdowns in the log backup pipeline.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 503, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Endpoint Status", "group": "Backup Log", "used_for": "Used for monitoring whether log backup is enabled on TiKV instances. Helps verify backup configuration status and quickly identify instances where log backup functionality may be disabled or misconfigured.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD Client CMD Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed PD client commands from TiDB instances, broken down by command type. Essential for identifying communication issues between TiDB and PD, helping diagnose cluster coordination problems and potential network or PD availability issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 507, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flushed Files (Last 30m) Per Host", "group": "Backup Log", "used_for": "Used for monitoring the number of log backup files flushed to storage per TiKV instance over the last 30 minutes. Helps diagnose backup performance issues, identify instances with abnormal flush rates, and detect potential bottlenecks in the log backup pipeline.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD TSO RPC Duration", "group": "PD Client", "used_for": "Used for monitoring the tail latency of TSO (Timestamp Oracle) RPC requests from TiDB to PD at p99, p999, and p9999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication issues that impact overall cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99, 999, 9999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 201, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Statement Lock Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of lock keys acquired per statement in TiDB transactions. Helps identify statements that acquire excessive locks, which can lead to lock contention, transaction conflicts, and degraded cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 159, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Async TSO Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of asynchronous timestamp oracle (TSO) requests from TiDB to PD at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication issues that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 515, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory Of Events", "group": "Backup Log", "used_for": "Used for monitoring the heap memory consumption of backup log events in TiKV instances. Essential for tracking memory usage during log backup operations and preventing out-of-memory issues in backup workers.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 57, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Scan Keys Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per distributed SQL query at different percentiles (p50, p90, max). Helps identify queries with excessive key scans that may indicate missing indexes, inefficient query plans, or full table scans causing performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 311, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Mpp Coordinator Counter", "group": "Executor", "used_for": "Used for monitoring the rate of MPP (Massively Parallel Processing) coordinator operations in TiDB, categorized by operation type. Essential for tracking the activity and throughput of MPP query coordination, helping diagnose issues with distributed analytical query execution and identifying bottlenecks in the MPP framework.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_mpp_coordinator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 12, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Distsql Duration", "group": "Distsql", "used_for": "Used for monitoring the latency distribution of distributed SQL (DistSQL) query execution in TiDB, tracking p50, p90, p99, and p999 percentiles by query type. Essential for diagnosing performance bottlenecks in distributed query processing, identifying slow coprocessor requests to TiKV, and detecting tail latency issues that impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572011, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Estimate TSO RTT Latency", "group": "PD Client", "used_for": "Used for monitoring the estimated round-trip time (RTT) latency between TiDB instances and PD when requesting TSO (Timestamp Oracle) allocations. Critical for diagnosing timestamp allocation delays that can impact transaction performance and identifying network or PD service issues affecting distributed transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "pd_client_request_estimate_tso_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 508, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flush Times (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the frequency of backup log flush operations in TiKV over the last 30 minutes. Helps diagnose backup performance issues by tracking how often log data is being flushed to storage, which is critical for understanding backup throughput and identifying potential bottlenecks in the log backup pipeline.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 247, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD HTTP Request OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second of HTTP requests from TiDB to PD (Placement Driver). Helps identify communication patterns between TiDB and PD, diagnose potential PD client bottlenecks, and detect abnormal request spikes that may indicate metadata access issues or cluster coordination problems.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 510, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flush Files (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the number of log backup files flushed to external storage in the last 30 minutes across TiKV instances. Helps track backup progress and identify potential issues with log backup file generation or storage operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Partial Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of partial results returned per DistSQL query at p50, p90, and p100 percentiles in TiDB. Helps diagnose query execution efficiency and identify queries that generate excessive partial results, which may indicate suboptimal query plans or data distribution issues affecting distributed query performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 14, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Distsql QPS", "group": "Distsql", "used_for": "Used for monitoring the queries per second (QPS) of distributed SQL requests in TiDB, broken down by coprocessor type (e.g., table scan, index scan). Essential for identifying workload patterns and diagnosing performance issues related to distributed query execution between TiDB and TiKV.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (copr_type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 60, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Distsql Partial QPS", "group": "Distsql", "used_for": "Used for monitoring the rate of partial distributed SQL scan operations per second in TiDB. Helps identify when queries are performing partial scans (scanning subset of data) versus full table scans, which is useful for diagnosing query execution patterns and potential performance issues related to data distribution or predicate pushdown effectiveness.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 312, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Mpp Coordinator Latency", "group": "Executor", "used_for": "Used for monitoring the latency of MPP (Massively Parallel Processing) coordinator operations in TiDB at different percentiles (p50, p75, p95). Essential for diagnosing performance issues in TiFlash analytical queries, particularly when investigating slow OLAP workloads or MPP task coordination bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 75, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.75, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 513, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial Scan Generate Event Throughput", "group": "Backup Log", "used_for": "Used for monitoring the data throughput rate during initial backup scanning in TiKV's log backup feature. Helps diagnose backup performance issues and track how quickly TiKV is scanning and generating backup events during the initial backup phase.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 509, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Total Flushed Size (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total amount of backup log data flushed to storage by TiKV instances over the last 30 minutes. Helps assess backup throughput, identify backup performance bottlenecks, and verify that log backup is progressing normally across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Scan Keys Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of keys scanned in partial results during distributed SQL execution at p80, p90, and p100 percentiles. Helps diagnose query efficiency issues by identifying queries that scan excessive keys, which may indicate missing indexes or inefficient query plans causing high latency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 77, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD TSO Wait Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the latency of TiDB instances waiting to obtain timestamps from PD's TSO (Timestamp Oracle) service at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD service bottlenecks that impact overall cluster performance, especially when experiencing slow transaction commits or high latency in distributed transactions.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"wait\"}[2m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"wait\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 339, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stale Region From PD", "group": "PD Client", "used_for": "Used for monitoring the rate at which TiDB receives stale region information from PD (Placement Driver). High values indicate region cache inconsistencies, which can cause TiDB to retry requests and impact query performance, signaling potential issues with PD's region metadata updates or network delays between TiDB and PD.", "component": "TIDB", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_region_from_pd{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Coprocessor Seconds 999", "group": "Distsql", "used_for": "Used for monitoring the 99.9th percentile latency of coprocessor requests sent from TiDB to TiKV. Critical for identifying tail latency in distributed query execution, particularly when diagnosing slow analytical queries or table scan performance issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\", type=\"Cop\"}[2m])) by (le,instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 248, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD HTTP Request Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed HTTP requests from TiDB to PD (Placement Driver). Critical for identifying communication issues between TiDB and PD, which can indicate network problems, PD overload, or cluster coordination failures that may impact query routing and metadata operations.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result!~\"200.*\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result!~\"200.*\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572019, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Circuit Breaker Event", "group": "PD Client", "used_for": "Used for monitoring PD client circuit breaker events in TiDB instances, tracking when circuit breakers open or close due to PD communication issues. Essential for diagnosing connection stability problems between TiDB and PD, helping identify when TiDB temporarily stops sending requests to PD to prevent cascading failures.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_request_circuit_breaker_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (name, event)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 175, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Coprocessor Cache", "group": "Distsql", "used_for": "Used for monitoring the hit rate and effectiveness of TiDB's coprocessor cache, which stores results from TiKV coprocessor requests. Helps identify cache efficiency and opportunities to reduce redundant distributed queries by tracking cache hits versus misses.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 27, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Schema Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing slow DDL execution, schema synchronization delays, and identifying performance bottlenecks when TiDB instances reload metadata from TiKV.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 246, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "PD HTTP Request Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of HTTP requests from TiDB to PD (Placement Driver) at p90, p99, and p999 percentiles. Critical for diagnosing communication issues between TiDB and PD, such as metadata retrieval delays, TSO allocation slowness, or network problems that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 315, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiDB Wait TSO Future Duration", "group": "PD Client", "used_for": "Used for monitoring the wait time when TiDB receives a timestamp (TSO) from PD that is in the future and must wait for local time to catch up. Critical for diagnosing clock synchronization issues between TiDB and PD nodes, which can cause transaction delays and latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_ts_future_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le) / sum(rate(tidb_tikvclient_ts_future_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 514, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Abnormal Checkpoint TS Lag", "group": "Backup Log", "used_for": "Used for monitoring the time lag between the current time and the last backup checkpoint timestamp in TiDB's log backup feature. Critical for detecting backup delays or failures that could impact point-in-time recovery capabilities and data protection SLAs.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 511, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Backup Log", "used_for": "Used for monitoring CPU usage of TiKV's backup log threads during continuous data backup operations. Essential for identifying resource bottlenecks in the backup stream process and ensuring backup tasks don't impact cluster performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 222, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Request Forwarded Status", "group": "PD Client", "used_for": "Used for monitoring whether PD client requests from TiDB are being forwarded to other PD nodes. Helps diagnose PD leader availability issues and identify when TiDB instances cannot directly reach the PD leader, requiring request forwarding through follower nodes.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "pd_client_request_forwarded_status" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 518, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Fatal Errors", "group": "Backup Log", "used_for": "Used for monitoring fatal errors in TiKV's backup log functionality, tracking error occurrences by type and instance. Critical for identifying backup failures and data protection issues that require immediate attention to prevent data loss.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 517, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Errors", "group": "Backup Log", "used_for": "Used for monitoring error rates in TiKV's log backup process, broken down by error type and instance. Essential for quickly identifying backup failures, diagnosing which TiKV nodes are experiencing issues, and understanding the nature of backup-related problems affecting data protection.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 29, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Schema Lease Error OPM", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema lease errors per minute in TiDB instances. Critical for detecting schema synchronization issues between TiDB servers, which can cause query failures or inconsistent metadata when the distributed schema lease mechanism fails.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Backoff Duration", "group": "KV Errors", "used_for": "Used for monitoring the duration TiDB spends in backoff retry mechanisms when encountering TiKV client errors (such as region unavailable, lock conflicts, or server busy). Critical for diagnosing performance degradation caused by transient failures, resource contention, or cluster instability that triggers automatic retry logic.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 516, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Observed Region Count", "group": "Backup Log", "used_for": "Used for monitoring the number of regions being observed by the backup log feature on each TiKV instance. Essential for verifying backup coverage and diagnosing issues where certain regions may not be included in log backup operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Lock Resolve OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of lock resolution operations in TiDB's transaction processing. Essential for diagnosing transaction conflicts and identifying when the cluster is experiencing high contention, which can impact write performance and transaction latency.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 11, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiClient Region Error OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of region-related errors encountered by TiDB when accessing TiKV, broken down by error type (including server_is_busy). Essential for diagnosing region availability issues, hotspot problems, and TiKV overload conditions that impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 53, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Backoff OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of KV client backoff operations in TiDB, categorized by backoff type. Essential for identifying when TiDB encounters transient errors (like region unavailable, lock conflicts, or server busy) that trigger retry mechanisms, helping diagnose cluster stability and contention issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 251, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Schema Action Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing slow DDL execution, schema synchronization delays, and identifying when schema changes are impacting cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, action))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 520, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Flush Duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to flush backup log data to external storage in TiKV. Critical for diagnosing backup performance issues and identifying bottlenecks in the log backup pipeline that could impact RPO (Recovery Point Objective).", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 521, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial scanning duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to complete initial data scanning during TiKV log backup operations. Critical for diagnosing backup performance issues and identifying slow initial scan phases that may delay backup readiness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Schema OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema loading operations in TiDB instances, broken down by operation type. Essential for diagnosing schema synchronization issues, DDL performance problems, and identifying instances experiencing excessive schema reload activity that could impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (instance,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 519, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Checkpoint TS of Tasks", "group": "Backup Log", "used_for": "Used for monitoring the checkpoint timestamp lag of backup log tasks in TiKV. Helps identify backup delays by comparing the last checkpoint position against current time, critical for ensuring point-in-time recovery capabilities and detecting backup task stalls.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Schema Cache OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema cache operations in TiDB, broken down by action and type. Helps diagnose schema metadata access patterns and identify potential bottlenecks in schema loading or cache invalidation that could impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infocache_counters{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (action,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 172, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Request OPS", "group": "KV Request", "used_for": "Used for monitoring the operations per second of different KV request types sent from TiDB to TiKV. Essential for identifying which request types (Get, Scan, Prewrite, Commit, etc.) are experiencing high throughput or sudden traffic changes that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 223, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Replica Selector Failure Per Second", "group": "KV Errors", "used_for": "Used for monitoring the rate of replica selector failures in TiDB's TiKV client, categorized by failure type. Critical for diagnosing issues with TiDB's ability to select appropriate TiKV replicas for read/write operations, which can indicate network problems, region unavailability, or load balancing issues.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572002, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Lease Duration", "group": "Schema Load", "used_for": "Used for monitoring the remaining time until TiDB's schema lease expires. Critical for diagnosing schema synchronization issues and detecting potential schema lease expiration problems that could cause DDL operations to fail or stall.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} - time()" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 522, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Convert Raft Event duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent converting Raft events during TiKV backup log operations. Critical for diagnosing performance bottlenecks in the log backup pipeline, particularly when backup lag increases or event processing slows down.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Request Duration 99 By Store", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of KV requests to each TiKV store from TiDB instances. Essential for identifying performance bottlenecks and uneven load distribution across individual TiKV stores, helping diagnose slow queries caused by specific store issues.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le, store))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 157, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Load Data From Cached Table Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile latency when loading data from TiDB's cached tables. Critical for diagnosing performance issues with cached table access and identifying when cache loading operations are taking longer than expected, which could impact query performance for applications using table caching features.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 523, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Wait for Lock Duration", "group": "Backup Log", "used_for": "Used for monitoring the time TiKV backup log operations spend waiting to acquire locks. Critical for diagnosing backup performance bottlenecks caused by lock contention, especially when backup tasks are slow or experiencing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Request Duration 99 By Type", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of different KV request types (Get, Scan, Prewrite, Commit, etc.) sent from TiDB to TiKV. Essential for identifying which specific request types are experiencing high latency and diagnosing performance bottlenecks in the TiDB-TiKV communication layer.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le,type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572018, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Infoschema V2 Cache Table Count", "group": "Schema Load", "used_for": "Used for monitoring the number of tables cached in TiDB's Infoschema V2 cache. Helps diagnose schema metadata memory usage and identify potential cache efficiency issues when troubleshooting schema-related performance problems or memory pressure.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_infoschema_v2_cache_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572014, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Infoschema V2 Cache Size", "group": "Schema Load", "used_for": "Used for monitoring the memory usage of TiDB's Infoschema V2 cache against its configured limit. Essential for diagnosing schema metadata caching efficiency and identifying potential memory pressure issues when the cache size approaches or exceeds its limit.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_infoschema_v2_cache_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "tidb_domain_infoschema_v2_cache_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 219, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Request Forwarding OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests being forwarded between TiKV stores in TiDB. Essential for diagnosing load balancing issues, identifying hot regions being redirected, and troubleshooting request routing problems when stores forward requests to other stores due to leader transfers or region migrations.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (from_store, to_store, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572012, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Infoschema V2 Cache Operation", "group": "Schema Load", "used_for": "Used for monitoring the performance of TiDB's Infoschema V2 cache operations, tracking cache hits, misses, and overall hit rate. Essential for diagnosing schema metadata access efficiency and identifying when schema cache tuning is needed to reduce metadata loading overhead.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"hit\"}[2m]))/(sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"hit\"}[2m]))+sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"miss\"}[2m])))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 524, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Command Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of batch sizes when TiKV processes key-value pairs during backup log operations. Helps identify batching efficiency and diagnose potential performance issues in log backup throughput by showing whether commands are being processed in optimal batch sizes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 526, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent writing backup log data to temporary files in TiKV. Critical for diagnosing performance bottlenecks in the log backup process, especially when backup operations are slow or experiencing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 164, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Region Cache OK OPS", "group": "KV Request", "used_for": "Used for monitoring the successful operations per second on TiDB's region cache, broken down by operation type (e.g., hit, miss, invalidate). Essential for evaluating region cache effectiveness and diagnosing issues with region location lookups that could impact query routing performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 525, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Save to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to save backup log events to temporary files in TiKV. Critical for diagnosing performance bottlenecks in the log backup pipeline, especially when backup operations are slow or experiencing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 250, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Region Cache Error OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of region cache operation errors in TiDB's TiKV client, broken down by error type. Critical for diagnosing region metadata issues, stale cache problems, and connectivity issues between TiDB and PD that can cause query failures or performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"err\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 528, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Internal Message Type", "group": "Backup Log", "used_for": "Used for monitoring the distribution and rate of internal message types processed by TiKV's backup log actor. Helps diagnose backup log performance issues by identifying which internal message operations are most frequent or experiencing bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 190, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "OPS", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per second in TiDB, broken down by operation type (CREATE, ALTER, DROP, etc.). Essential for tracking DDL workload intensity and identifying periods of heavy schema changes that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 220, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "KV Request Forwarding OPS By Type", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded by TiDB to other TiKV nodes, broken down by request type and result status. Helps diagnose request routing issues, identify forwarding patterns, and detect potential problems with stale read replicas or follower read configurations.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 527, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "System Write Call Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of system write calls during TiKV log backup operations. Essential for diagnosing backup performance issues and identifying I/O bottlenecks that may impact log backup throughput and reliability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 306, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stale Read Hit/Miss OPS", "group": "KV Request", "used_for": "Used for monitoring the throughput of stale read requests in TiDB, showing the rate of successful hits versus misses when reading historical data. Helps diagnose stale read feature effectiveness and identify issues with reading from local replicas or follower nodes.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 307, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stale Read Req OPS", "group": "KV Request", "used_for": "Used for monitoring the throughput of stale read requests in TiDB, broken down by request type. Helps track the usage and performance of stale read features, which allow reading slightly outdated data for improved read scalability and reduced latency.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_req_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 64, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Deploy Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL syncer deployment operations in TiDB. Critical for diagnosing slow DDL execution and identifying issues with schema change synchronization across the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 532, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Operation", "group": "Backup Log", "used_for": "Used for monitoring the rate of RocksDB operations during the initial scan phase of TiKV log backup. Helps diagnose I/O performance and bottlenecks when backup tasks are scanning and reading data from RocksDB storage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 308, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stale Read Req Traffic", "group": "KV Request", "used_for": "Used for monitoring the network traffic volume of stale read requests in TiDB, broken down by result status and direction (sent/received). Essential for tracking the bandwidth consumption of stale read operations and identifying potential network bottlenecks or unusual traffic patterns when using stale read features for improved read performance.", "component": "TIDB", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, direction)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 530, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile latency of internal message handling in TiKV's backup log subsystem. Critical for diagnosing performance bottlenecks in log backup operations and identifying when internal actor processing becomes slow.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 529, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration $optional_quantile", "group": "Backup Log", "used_for": "Used for monitoring the latency of internal message handling in TiKV's backup log actor system at specified quantiles. Essential for diagnosing performance bottlenecks in the log backup pipeline and identifying slow internal operations that may impact backup efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Job Worker Operations Duration", "group": "DDL", "used_for": "Used for monitoring the 99th percentile latency of DDL worker operations in TiDB, broken down by operation type and action. Critical for diagnosing slow DDL execution issues such as table creation, index building, or schema modifications that may impact application deployment or maintenance operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (le, type, action, result))", "sum(increase(tidb_ddl_worker_operation_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type, action, result)/sum(increase(tidb_ddl_worker_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type, action, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 65, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Sync Schema Version Operations Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL schema version synchronization operations in TiDB, including both the DDL owner's syncer handling time and self-version update duration. Critical for diagnosing DDL performance issues and identifying delays in schema change propagation across the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(.99, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type, result))", "histogram_quantile(.99, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, result))", "rate(tidb_ddl_owner_handle_syncer_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]) / rate(tidb_ddl_owner_handle_syncer_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])", "sum(rate(tidb_ddl_update_self_ver_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result) / sum(rate(tidb_ddl_update_self_ver_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 62, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Waiting Job Count", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs waiting in the queue across TiDB instances. Critical for identifying DDL job backlogs and potential schema change bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572015, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TableByName API Duration", "group": "Schema Load", "used_for": "Used for monitoring the latency of TiDB's TableByName API operations, which retrieve table metadata from the information schema by table name. Critical for diagnosing slow schema lookups that can impact query planning and execution, especially when applications frequently access table metadata or during schema-heavy workloads.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type) /\nsum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type)", "histogram_quantile(0.999, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 55, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "DDL META OPM", "group": "DDL", "used_for": "Used for monitoring the rate of DDL (Data Definition Language) metadata operations per minute in TiDB. Helps track DDL activity frequency and identify periods of schema changes or potential DDL bottlenecks affecting cluster performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 263, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "RPC Layer Latency", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile network latency between TiDB and TiKV nodes during RPC communication. Critical for diagnosing network-related performance issues, identifying slow TiKV stores, and distinguishing between storage engine delays versus network transmission delays in KV request processing.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store))", "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 9, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Execute Duration", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operation execution latency in TiDB, tracking both overall job handling duration and batch index addition performance at p95 and p99 percentiles. Critical for diagnosing slow schema changes, identifying DDL bottlenecks during table alterations or index creation, and understanding when DDL operations impact cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_ddl_handle_job_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 535, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Key Putting", "group": "Backup Log", "used_for": "Used for monitoring the rate of region checkpoint key operations during TiKV log backup metadata updates. Helps diagnose backup performance issues and identify bottlenecks in the checkpoint persistence process when backup lag increases.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 264, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "System Table Operations Duration", "group": "DDL", "used_for": "Used for monitoring the 95th percentile latency of DDL operations on TiDB system tables, including metadata updates and schema changes. Critical for diagnosing slow DDL execution and identifying bottlenecks in system table access that can impact overall cluster performance and schema modification operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", }[2m])) by (le, type))", "sum(rate(tidb_ddl_job_table_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tidb_ddl_job_table_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 533, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Trigger Reason", "group": "Backup Log", "used_for": "Used for tracking the reasons that trigger initial scanning operations in TiKV's backup log feature. Helps diagnose why backup processes are initiating full scans, which is essential for troubleshooting backup performance issues and understanding backup behavior patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 20, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD Client CMD OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second (OPS) of different PD client command types in TiDB, excluding TSO requests. Helps identify which PD operations (like GetRegion, GetStore) are most frequent and diagnose potential bottlenecks in TiDB's communication with PD.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"tso\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 262, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Region Duration", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of TiDB loading region metadata from the region cache. Critical for diagnosing slow KV request issues caused by region cache misses or PD communication delays, especially when queries experience unexpected latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 192, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Backfill Progress In Percentage", "group": "DDL", "used_for": "Used for monitoring the completion percentage of DDL backfill operations, including index creation and column modifications. Essential for tracking long-running DDL progress and estimating completion time during schema changes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}", "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"modify_column\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 534, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Task Status", "group": "Backup Log", "used_for": "Used for monitoring the status and progress of initial scanning tasks during TiKV backup log operations. Helps identify bottlenecks in the backup initialization phase by tracking pending scan tasks across different stages, essential for diagnosing slow backup startup or stalled initial scan processes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (stage) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD Client CMD Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed PD client commands in TiDB, broken down by command type. Essential for identifying communication issues between TiDB and PD, helping diagnose cluster metadata access problems and potential network or PD availability issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 35, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD Client CMD Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of PD client commands (excluding TSO operations) from TiDB instances at p90, p99, and p999 percentiles. Essential for diagnosing communication delays between TiDB and PD, helping identify network issues, PD overload, or metadata operation bottlenecks that impact query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 265, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Running Job Count By Worker Pool", "group": "DDL", "used_for": "Used for monitoring the number of DDL jobs currently executing in each worker pool within TiDB. Helps identify DDL concurrency bottlenecks and diagnose schema change performance issues when multiple DDL operations are queued or running simultaneously.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_running_job_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 337, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Client-side Slow Score", "group": "KV Request", "used_for": "Used for monitoring the slow score of each TiKV store as perceived by TiDB clients. Helps identify problematic TiKV stores experiencing performance degradation or network issues that may cause request slowdowns, enabling quick isolation of unhealthy nodes in the cluster.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_store_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 338, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiKV-side Slow Score", "group": "KV Request", "used_for": "Used for monitoring the slow score feedback from each TiKV store as perceived by TiDB clients. Helps identify which TiKV stores are experiencing performance degradation or responding slowly, enabling quick diagnosis of unbalanced load or problematic storage nodes.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_feedback_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (store)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572001, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Add Index Scan Rate", "group": "DDL", "used_for": "Used for monitoring the table scanning throughput (rows per second) during ADD INDEX operations in TiDB. Essential for evaluating index creation performance and identifying slow index building processes that may impact cluster resources.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 531, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Throughput", "group": "Backup Log", "used_for": "Used for monitoring the read throughput from RocksDB during the initial scan phase of TiKV log backup operations. Essential for diagnosing backup performance issues and identifying bottlenecks in the initial data scanning process when setting up log backup.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD TSO OPS", "group": "PD Client", "used_for": "Used for monitoring the throughput of TSO (Timestamp Oracle) requests from TiDB to PD, measured in operations per second. Essential for identifying TSO request bottlenecks, understanding transaction workload intensity, and diagnosing issues when applications experience timestamp allocation delays or PD client connection problems.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))", "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763573002, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Retryable Error", "group": "DDL", "used_for": "Used for monitoring the rate of retryable errors encountered during DDL operations in TiDB. Helps identify transient issues like schema conflicts or temporary resource constraints that cause DDL statements to retry, which may indicate contention or instability in schema changes.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_ddl_retryable_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 536, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Request Checkpoint Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the batch size of checkpoint requests during TiDB log backup operations. Helps identify backup efficiency issues and optimize batch processing by tracking how many log entries are processed together in each checkpoint request.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD TSO Wait Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the wait duration (p90, p99, p999) when TiDB requests TSO (Timestamp Oracle) from PD. Critical for diagnosing transaction latency issues caused by PD client bottlenecks or network delays between TiDB and PD servers.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 78, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD TSO RPC Duration", "group": "PD Client", "used_for": "Used for monitoring the tail latency of TSO (Timestamp Oracle) RPC requests from TiDB to PD at p99, p999, and p9999 percentiles. Critical for diagnosing transaction timestamp allocation delays that can impact overall transaction performance and identifying PD service bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99, 999, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 539, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Request Result", "group": "Backup Log", "used_for": "Used for monitoring the success and failure rates of backup log region requests in TiKV. Helps diagnose backup reliability issues by tracking request outcomes and identifying potential failures in the log backup process.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Backfill Data Rate", "group": "DDL", "used_for": "Used for monitoring the rate at which TiDB backfills data during DDL operations like adding indexes. Essential for tracking DDL progress and diagnosing slow index creation performance, helping identify whether backfill operations are proceeding at expected throughput rates.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_add_index_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 540, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Tick Duration $optional_quantile", "group": "Backup Log", "used_for": "Used for monitoring the latency of backup log advancement ticks in TiKV at specified percentiles. Essential for diagnosing performance issues in the log backup process and identifying when backup operations are experiencing delays or bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 541, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile duration of backup log advancement ticks in TiKV. Essential for diagnosing performance issues with log backup operations and identifying when the backup advancer is experiencing delays in processing checkpoint updates.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 322, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Task Status", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of task statuses in TiDB's distributed execution framework, broken down by task type. Helps identify stuck or failed distributed tasks (like DDL operations, import jobs, or analytics queries) and track overall task execution health across the cluster.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (status, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 329, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pending Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for monitoring the number of pending subtasks in TiDB's distributed execution framework, grouped by executor ID. Helps identify task scheduling bottlenecks and uneven workload distribution across executors when distributed tasks are queued but not yet executing.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", status=~\"pending\"}) by (exec_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 542, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Get Region Operation Count", "group": "Backup Log", "used_for": "Used for monitoring the rate of region metadata retrieval operations during TiDB log backup processes. Helps diagnose backup performance issues by tracking how frequently the backup advancer needs to fetch region information from PD or TiKV.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573003, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Add Index Backfill Import Speed", "group": "DDL", "used_for": "Used for monitoring the data import speed (bytes per second) during the add index backfill process in TiDB's distributed execution framework. Essential for tracking index creation performance and diagnosing slow index building operations, particularly when using the ingest mode for large-scale index additions.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"imported\"}[2m])) by(job_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 326, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Completed/Total Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for monitoring the progress of distributed task execution in TiDB's distributed execution framework by tracking completed versus total subtask counts. Essential for diagnosing task execution bottlenecks, identifying stuck or failed subtasks, and assessing overall distributed operation health during large-scale operations like DDL or data import tasks.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", status=~\"succeed|failed|canceled|reverted|revert_failed\"}) by (task_id, task_type)", "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (task_id, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 159, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Async TSO Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of asynchronous TSO (Timestamp Oracle) requests from TiDB to PD at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication bottlenecks that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 2, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Write Pipeline Duration", "group": "Duration", "used_for": "Used for monitoring the latency of TiKV's Raft write pipeline stages, including append log duration, request wait time, and apply wait time. Essential for diagnosing write performance bottlenecks and identifying which stage of the Raft consensus process is causing delays in data persistence.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 330, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "SubTask Running Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the execution duration of currently running subtasks in TiDB's distributed task execution framework. Helps identify long-running subtasks and diagnose performance bottlenecks in distributed operations like DDL, import, or analytics queries.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", status=~\"running\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 246, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD HTTP Request Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of HTTP requests from TiDB to PD (Placement Driver) at different percentiles (p90, p99, p999). Essential for diagnosing communication issues between TiDB and PD, identifying network bottlenecks, or detecting PD performance degradation that could impact metadata operations and transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572011, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Estimate TSO RTT Latency", "group": "PD Client", "used_for": "Used for monitoring the estimated round-trip time (RTT) latency between TiDB and PD when requesting TSO (Timestamp Oracle) allocations. Critical for diagnosing timestamp allocation delays that can impact transaction performance and identifying network or PD service issues affecting distributed transaction coordination.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_client_request_estimate_tso_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 222, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Request Forwarded Status", "group": "PD Client", "used_for": "Used for monitoring whether PD client requests from TiDB are being forwarded to other PD nodes. Helps diagnose PD leader issues, network problems, or cluster topology changes when requests cannot be served directly by the initially contacted PD node.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "pd_client_request_forwarded_status" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 545, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of threads across TiKV instances, showing how many threads are in different states (running, sleeping, waiting, etc.). Essential for diagnosing thread contention, resource saturation, and identifying instances with abnormal thread behavior that could indicate I/O bottlenecks or CPU pressure.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) ", "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 546, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput across TiKV threads, showing the top 20 threads by I/O bytes per second. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or abnormal disk activity patterns in TiKV instances.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Store size", "group": "Cluster", "used_for": "Used for monitoring the total disk space consumed by each TiKV store instance. Essential for tracking storage capacity usage, identifying stores approaching disk limits, and planning capacity expansion before running out of space.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all_used\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 331, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Subtask Pending Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the time subtasks spend in pending state before execution in TiDB's distributed task framework. Helps identify task scheduling bottlenecks and resource contention issues when subtasks are queued but not being processed.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", status=\"pending\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 248, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD HTTP Request Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed HTTP requests from TiDB to PD (Placement Driver). Essential for identifying communication issues between TiDB and PD, such as network problems, PD unavailability, or API errors that could impact cluster metadata operations and query routing.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m])) by (type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 7, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Capacity size", "group": "Cluster", "used_for": "Used for monitoring the total storage capacity configured for each TiKV instance. Essential for capacity planning and identifying instances approaching their storage limits before they run out of disk space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Available size", "group": "Cluster", "used_for": "Used for monitoring the total available disk space across TiKV instances in the cluster. Critical for capacity planning and preventing storage exhaustion issues that could impact cluster stability and write operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"available\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 543, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Try Advance Trigger Time", "group": "Backup Log", "used_for": "Used for monitoring the frequency of backup log advancement attempts in TiKV. Helps diagnose backup log processing activity and identify when the log backup advancer is actively triggering checkpoint updates.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 339, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stale Region From PD", "group": "PD Client", "used_for": "Used for monitoring the rate at which TiDB receives stale region information from PD (Placement Driver). Helps diagnose region cache inconsistencies and PD communication issues that can cause query routing errors or performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_region_from_pd{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 247, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "PD HTTP Request OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second of HTTP requests from TiDB to PD (Placement Driver). Helps identify communication patterns between TiDB and PD, diagnose PD client bottlenecks, and detect abnormal request rates that may indicate metadata access issues or cluster coordination problems.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 3, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Cop Read Duration", "group": "Duration", "used_for": "Used for monitoring the end-to-end latency of coprocessor read requests in TiKV, broken down into storage engine async request duration, coprocessor wait time, and actual request handling time. Essential for diagnosing slow table scan or index lookup performance issues and identifying whether bottlenecks occur in storage access, request queuing, or computation phases.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring TiKV cluster memory usage, comparing actual resident memory consumption against configured memory quotas. Essential for detecting memory pressure, potential OOM risks, and validating whether TiKV instances are operating within their allocated memory limits.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) by (instance) ", "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) ", "max((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) ", "min((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 554, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Mapped Allocation per Thread", "group": "Memory", "used_for": "Used for monitoring memory mapped allocations per thread in TiKV instances. Helps identify memory allocation patterns at the thread level and diagnose potential memory leaks or excessive memory usage by specific threads.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 552, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Allocated Bytes Rate per Thread", "group": "Memory", "used_for": "Used for monitoring the memory allocation rate per thread in TiKV instances. Helps identify memory allocation patterns, detect memory-intensive threads, and diagnose potential memory leaks or excessive allocation issues that could impact TiKV performance.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 315, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiDB Wait TSO Future Duration", "group": "PD Client", "used_for": "Used for monitoring the duration TiDB waits when a timestamp from PD is in the future compared to local time. Critical for diagnosing clock synchronization issues between TiDB and PD nodes that can cause transaction delays and latency spikes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_ts_future_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le) / sum(rate(tidb_tikvclient_ts_future_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 537, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Tick Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of backup log advancement ticks in TiKV. Helps diagnose performance issues with log backup operations and identify when the backup advancer is experiencing delays or bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 332, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Uncompleted Subtask Distribution On TiDB Nodes", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of incomplete distributed task subtasks across TiDB nodes. Helps identify workload imbalances and stuck subtasks in the distributed execution framework, essential for diagnosing task scheduling issues and node-level bottlenecks.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", status=~\"pending|running|failed|canceled|paused\"}) by (exec_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 550, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Allocator Stats", "group": "Memory", "used_for": "Used for monitoring memory allocation statistics in TiKV instances, breaking down memory usage by allocation type (e.g., allocated, active, resident). Essential for diagnosing memory leaks, understanding memory fragmentation, and troubleshooting OOM (out-of-memory) issues in TiKV nodes.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Inaccuracy Rate", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the distribution of statistics inaccuracy rates in TiDB's query optimizer at p50, p90, and p99 percentiles. Critical for identifying when table statistics become stale or inaccurate, which can lead to suboptimal query plans and performance degradation requiring statistics updates.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 90, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 328, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Slots Usage", "group": "Dist Execute Framework", "used_for": "Used for monitoring the resource utilization of TiDB's distributed task execution framework by comparing active slot usage against maximum available CPU cores (maxprocs). Essential for identifying resource contention, capacity planning, and diagnosing performance bottlenecks in distributed operations like DDL, import, and analytics tasks.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_disttask_used_slots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 547, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring voluntary context switches across TiKV threads, which occur when threads willingly yield the CPU (typically during I/O waits). High rates indicate threads are frequently waiting for I/O operations, helping diagnose I/O bottlenecks and thread scheduling efficiency issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 251, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Schema Action Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile latency of schema loading operations in TiDB, broken down by action type (e.g., reload, update). Critical for diagnosing DDL performance issues and identifying slow schema synchronization that can impact query execution and cluster stability.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, action))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572019, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Circuit Breaker Event", "group": "PD Client", "used_for": "Used for monitoring PD client circuit breaker events in TiDB, tracking when circuit breakers open or close due to PD communication issues. Essential for diagnosing connection stability problems between TiDB and PD, helping identify when TiDB instances are experiencing repeated failures or timeouts when accessing PD services.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_request_circuit_breaker_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, event)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 10, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring disk I/O utilization percentage across TiKV instances in the cluster. Essential for identifying storage bottlenecks, detecting overloaded disks, and diagnosing performance degradation caused by I/O saturation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 553, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Released Bytes Rate per Thread", "group": "Memory", "used_for": "Used for monitoring the rate of memory deallocation per thread in TiKV instances. Helps diagnose memory management efficiency and identify threads with abnormal memory release patterns that could indicate memory leaks or inefficient resource cleanup.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 551, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "group": "Memory", "used_for": "Used for monitoring the rate of memory allocation and deallocation across TiKV threads, showing allocated bytes as positive values and released bytes as negative values. Essential for identifying memory churn patterns, potential memory leaks, and understanding memory pressure during high-throughput operations.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Auto/Manual Analyze Queries Per Minute", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the frequency of automatic and manual ANALYZE operations per minute in TiDB. Essential for tracking statistics collection activity, diagnosing plan quality issues caused by stale statistics, and identifying when auto-analyze triggers too frequently or manual analyze operations are being executed.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_manual_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 555, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Arena Count", "group": "Memory", "used_for": "Used for monitoring the number of memory arenas allocated by TiKV's memory allocator (jemalloc) per instance. Helps diagnose memory fragmentation issues and abnormal memory allocation patterns that could indicate memory leaks or inefficient memory usage.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "MBps", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the aggregate throughput (in megabytes per second) of TiKV storage engine I/O operations, including WAL writes, disk reads, and in-memory engine operations. Essential for identifying storage bottlenecks, understanding cluster-wide I/O patterns, and diagnosing performance issues related to disk bandwidth saturation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 548, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches across TiKV threads, which occur when the OS forcibly preempts threads due to CPU contention or time slice expiration. High rates indicate CPU saturation or resource competition, helping diagnose performance degradation caused by insufficient CPU resources or thread scheduling issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 29, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Schema Lease Error OPM", "group": "Schema Load", "used_for": "Used for monitoring schema lease errors per minute in TiDB instances. Critical for detecting schema synchronization issues between TiDB servers, which can cause query failures or inconsistent metadata when the schema lease mechanism fails to maintain consistency across the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 46, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Auto/Manual Analyze Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the duration of automatic and manual ANALYZE operations at p80 and p95 percentiles. Essential for diagnosing slow statistics collection that can impact query plan quality and identifying when ANALYZE jobs are taking longer than expected, potentially affecting cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"AnalyzeTable\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"AnalyzeTable\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 233, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Healthy Distribution", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the health distribution of table statistics across different types (table/index) in TiDB. Critical for identifying stale or missing statistics that can lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "avg(tidb_statistics_stats_healthy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 71, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Pseudo Estimation OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of queries using pseudo statistics (statistics generated without actual table analysis) in TiDB. High values indicate tables lacking proper statistics, which can lead to suboptimal query plans and performance degradation requiring ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the distribution of Raft leader regions across TiKV instances in the cluster. Helps identify leader imbalance issues that can cause uneven load distribution and potential performance hotspots.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Schema Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing performance issues related to DDL operations and schema synchronization, especially when experiencing slow query execution or metadata update delays.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 93, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Update Stats OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of statistics update operations in TiDB, broken down by update type. Essential for tracking the frequency of table statistics refreshes and diagnosing issues with auto-analyze jobs or manual statistics collection that could impact query plan quality.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "QPS", "group": "Cluster", "used_for": "Used for monitoring the total queries per second (QPS) handled by TiKV instances across the cluster, excluding garbage collection operations. Essential for tracking overall TiKV workload intensity, identifying traffic spikes, and correlating performance issues with request volume changes.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring CPU utilization across TiKV instances in the cluster, showing actual CPU usage against allocated quota limits. Essential for identifying CPU bottlenecks, resource saturation, and determining whether TiKV nodes need vertical scaling or load rebalancing.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n [$__rate_interval]\n)) by (instance) ", "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) ", "max((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) ", "min((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Schema Cache OPS", "group": "Schema Load", "used_for": "Used for monitoring the operations per second of TiDB's schema cache, broken down by action and type. Helps diagnose schema metadata access patterns and identify potential bottlenecks in schema loading or cache invalidation operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infocache_counters{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (action,type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 308, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stale Read Req Traffic", "group": "KV Request", "used_for": "Used for monitoring the network traffic volume of stale read requests in TiDB, broken down by result status and direction (sent/received). Essential for tracking the usage and performance of stale read features, which allow reading slightly outdated data to reduce latency and load on leader replicas.", "component": "TIDB", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (result, direction)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 229, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Sync Load QPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the throughput of synchronous statistics loading operations in TiDB, including total load requests, timeouts, and deduplicated requests. Essential for diagnosing statistics loading performance issues and identifying when queries are blocked waiting for statistics to be loaded synchronously.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_dedup_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 13, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Errps", "group": "Cluster", "used_for": "Used for monitoring the rate of errors occurring in TiKV instances, including gRPC message failures, PD heartbeat no-ops, and critical errors. Essential for quickly identifying cluster health issues, communication problems between components, and detecting critical failures that require immediate attention.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Schema OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema loading operations in TiDB, broken down by instance and operation type. Essential for diagnosing schema synchronization issues, DDL performance problems, and identifying instances experiencing excessive schema reload activity that could impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 157, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Load Data From Cached Table Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile latency when loading data from TiDB's cached tables feature. Critical for diagnosing performance issues with table cache operations and identifying instances where cached table access is slower than expected.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 230, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Sync Load Latency P9999", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99.99th percentile latency of synchronous statistics loading operations in TiDB, including both overall sync load time and stats reading time. Critical for diagnosing extreme tail latency issues in query optimization when TiDB needs to load table statistics on-demand, which can impact query planning performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572002, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Lease Duration", "group": "Schema Load", "used_for": "Used for monitoring the remaining time until TiDB's schema lease expires. Critical for diagnosing schema synchronization issues and detecting potential schema lease expiration problems that could cause DDL operations to fail or stall.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} - time()" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 15, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the total number of regions and buckets distributed across TiKV instances in the cluster. Essential for tracking data distribution balance, identifying region hotspots, and diagnosing uneven load distribution that could impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) ", "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"buckets\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 235, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Cache OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per second on TiDB's statistics cache, broken down by operation type (hits, misses, evictions). Essential for diagnosing query optimizer performance issues and determining if statistics cache sizing is adequate for the workload.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_stats_cache_op{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 557, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Background Task Total Wait Duration", "group": "Resource Control", "used_for": "Used for monitoring the total wait duration of background tasks in TiKV's resource control system. Helps identify resource contention and scheduling delays when background tasks are queued, which can impact overall cluster performance and throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, resource_group) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572014, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Infoschema V2 Cache Size", "group": "Schema Load", "used_for": "Used for monitoring the memory usage of TiDB's InfoSchema V2 cache against its configured limit. Essential for diagnosing schema metadata memory consumption issues and determining if cache size adjustments are needed to optimize schema loading performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_infoschema_v2_cache_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "tidb_domain_infoschema_v2_cache_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 558, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Priority Quota Limit", "group": "Resource Control", "used_for": "Used for monitoring the I/O quota limits assigned to different priority levels in TiKV's resource control system. Essential for verifying resource allocation configurations and diagnosing whether workload throttling is due to reaching configured priority limits.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, priority) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 561, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Status API Request (op/s)", "group": "Status Server", "used_for": "Used for monitoring the rate of requests per second to TiKV's status API endpoints. Helps identify unusual access patterns to diagnostic endpoints and potential issues with monitoring or health check systems querying TiKV status information.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 18, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors occurring in TiKV instances. Essential for detecting severe system issues that require immediate attention, such as data corruption, storage failures, or critical internal errors that could impact cluster stability and data integrity.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 16, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring how long each TiKV instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and correlate service disruptions with specific TiKV node restarts during troubleshooting.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",component=~\".*(tikv|copr).*\"}\n \n)) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572018, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Infoschema V2 Cache Table Count", "group": "Schema Load", "used_for": "Used for monitoring the number of cached tables in TiDB's InfoSchema V2 cache. Helps diagnose schema metadata caching efficiency and identify potential memory pressure issues when the cache grows excessively large or fails to maintain expected table counts.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_infoschema_v2_cache_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 234, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Cache Cost", "group": "Statistics & Plan Management", "used_for": "Used for monitoring TiDB's statistics cache memory usage, tracking both current memory consumption and configured capacity limits. Essential for diagnosing statistics-related memory pressure and determining if cache size adjustments are needed to optimize query plan generation performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", type=\"track\"}", "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", type=\"capacity\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 560, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Status API Request Duration", "group": "Status Server", "used_for": "Used for monitoring the latency of TiKV status server API requests at p99 and p9999 percentiles. Essential for diagnosing performance issues with TiKV's HTTP status endpoints, which are used for health checks, metrics collection, and cluster management operations.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 563, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiKV instances. Helps track encryption key management and identify potential issues with key storage capacity or key rotation processes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 19, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring the rate of \"server busy\" errors in TiKV, including scheduler overload, channel congestion, and coprocessor queue saturation. Critical for diagnosing when TiKV instances are overwhelmed by excessive workload or resource contention, helping identify capacity bottlenecks and the need for scaling or load balancing.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "db", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 21, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiKV's Raftstore layer. Critical for detecting storage engine errors, consensus failures, and data replication issues that could impact cluster stability and data consistency.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572006, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Loading Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile duration of statistics loading operations in TiDB. Critical for diagnosing slow query plan generation issues caused by delayed statistics loading, which can impact query optimization and overall database performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_load_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 20, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiKV instances, which indicate communication or coordination issues between TiKV nodes and PD. Essential for diagnosing cluster stability problems, network failures, or PD connectivity issues that could impact data replication and cluster health.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance, store_id) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572015, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TableByName API Duration", "group": "Schema Load", "used_for": "Used for monitoring the latency of TiDB's TableByName API calls, which retrieve table metadata from the information schema by table name. Critical for diagnosing slow schema lookups that can impact query planning performance, especially when applications frequently access table metadata or experience delays in query compilation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) /\nsum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)", "histogram_quantile(0.999, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_infoschema_table_by_name_duration_nanoseconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 236, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Plan Replayer Task OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring Plan Replayer task execution rates and registration status in TiDB. Helps track the success/failure of plan capture and dump operations, essential for diagnosing issues with SQL plan collection and troubleshooting query performance problems through plan replay functionality.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)", "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"capture\"}[2m])) by (result)", "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 237, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Historical Stats OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per minute (OPM) of historical statistics generation and dumping in TiDB. Helps diagnose issues with statistics collection processes by tracking success/failure rates of stats generation and persistence operations, which are critical for query optimizer performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"generate\"}[2m])) by (result)", "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 22, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring the rate of scheduler errors in TiKV, including snapshot errors, prepare write errors, and other transaction scheduling failures. Critical for diagnosing transaction processing issues, write conflicts, and storage layer problems that prevent successful query execution.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 564, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiKV's storage engine across instances. Helps verify encryption-at-rest is working correctly and track the volume of encrypted data files when investigating encryption-related issues or compliance requirements.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 566, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the size of encryption metadata files in TiKV instances. Helps track metadata storage growth and identify potential issues with encryption key management or metadata accumulation that could impact disk usage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV instances. Essential for identifying issues with coprocessor operations (table scans, aggregations, etc.) that could indicate data corruption, resource exhaustion, or query execution problems affecting analytical workloads.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "New ETCD Session Duration 95", "group": "Owner", "used_for": "Used for monitoring the 95th percentile latency of establishing new etcd sessions in TiDB's owner election mechanism. Critical for diagnosing etcd connectivity issues and owner election delays that can impact DDL operations and cluster coordination.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 567, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Encrypt/decrypt data nanos", "group": "Encryption", "used_for": "Used for monitoring the time spent on encrypting and decrypting data in TiKV's RocksDB storage engine when encryption-at-rest is enabled. Essential for diagnosing encryption overhead and performance impact on I/O operations, particularly when investigating slower-than-expected read/write performance in encrypted clusters.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Execute Duration", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operation execution latency in TiDB, tracking both overall job handling duration and batch index addition performance at p95 and p99 percentiles. Critical for diagnosing slow schema changes, identifying DDL bottlenecks, and understanding the impact of ALTER TABLE or CREATE INDEX operations on cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_ddl_handle_job_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 265, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Running Job Count By Worker Pool", "group": "DDL", "used_for": "Used for monitoring the number of DDL jobs currently running in each worker pool across TiDB instances. Helps identify DDL workload distribution and potential bottlenecks when multiple schema changes are executing concurrently.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_running_job_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572012, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Infoschema V2 Cache Operation", "group": "Schema Load", "used_for": "Used for monitoring the performance of TiDB's Infoschema V2 cache operations, including hit/miss rates and operation types. Essential for diagnosing metadata access efficiency and identifying when schema information lookups are causing performance bottlenecks due to low cache hit rates.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hit\"}[2m]))/(sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hit\"}[2m]))+sum(rate(tidb_domain_infoschema_v2_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"miss\"}[2m])))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 24, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV instances. Critical for detecting communication errors between TiKV nodes or with TiDB/PD components, which can indicate network issues, resource exhaustion, or service degradation.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572008, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Binding Cache Memory Usage", "group": "Statistics & Plan Management", "used_for": "Used for monitoring memory consumption of TiDB's SQL binding cache, which stores execution plan bindings. Essential for tracking cache memory usage against configured limits to prevent out-of-memory issues and optimize plan binding performance.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_mem_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}", "tidb_server_binding_cache_mem_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 26, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions without a leader on each TiKV instance. Critical for detecting split-brain scenarios, network partitions, or leader election failures that can cause service unavailability and data access issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572010, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Number Of Bindings In Cache", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the number of SQL plan bindings currently stored in TiDB's cache. Helps track plan binding usage and diagnose issues when execution plans aren't being enforced as expected due to cache capacity or eviction problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_num_bindings{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 25, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region transfers or drops in TiKV instances. Critical for detecting leadership instability, network partitions, or node failures that cause frequent leader changes and potential service disruptions.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 568, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "**Used For:**\n\nUsed for monitoring the latency of reading and writing encryption metadata files in TiKV's encryption-at-rest feature. Critical for diagnosing performance issues with encryption key management operations, especially when experiencing slow I/O or degraded encryption performance at p99 and p9999 percentiles.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 65, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Sync Schema Version Operations Duration", "group": "DDL", "used_for": "Used for monitoring the latency of schema version synchronization operations in TiDB's DDL module, including both the DDL owner's syncer handling duration and self-version update duration at the 99th percentile. Critical for diagnosing slow DDL execution and schema synchronization delays across the cluster, especially when DDL operations appear stuck or take longer than expected.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(.99, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))", "histogram_quantile(.99, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, result))", "rate(tidb_ddl_owner_handle_syncer_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_ddl_owner_handle_syncer_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "sum(rate(tidb_ddl_update_self_ver_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result) / sum(rate(tidb_ddl_update_self_ver_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 28, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Log Replication Rejected", "group": "Errors", "used_for": "Used for monitoring the rate of rejected Raft log append operations in TiKV. Indicates when TiKV nodes are rejecting replication requests, typically due to resource constraints or backpressure, which can lead to replication lag and cluster instability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572005, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Stats Meta/Usage Updating Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile latency of statistics metadata and usage information updates in TiDB. Critical for diagnosing performance issues related to statistics collection and maintenance, which can impact query optimization and overall database performance when updates become slow.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_usage_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 264, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "System Table Operations Duration", "group": "DDL", "used_for": "Used for monitoring the 95th percentile latency of DDL operations on system tables in TiDB, broken down by operation type. Critical for diagnosing slow DDL execution and identifying bottlenecks in schema changes that interact with internal metadata tables.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", }[2m])) by (le, type))", "sum(rate(tidb_ddl_job_table_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tidb_ddl_job_table_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 50, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "AutoID QPS", "group": "Meta", "used_for": "Used for monitoring the rate of auto-increment ID allocation operations per second in TiDB. Helps identify workload patterns involving auto-increment columns and diagnose potential bottlenecks in ID generation during high-volume insert operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 64, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Deploy Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL syncer deployment operations in TiDB. Critical for diagnosing delays in DDL synchronization setup, which can impact schema change propagation and cluster coordination during DDL operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 62, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Waiting Job Count", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs queued and waiting to be executed in TiDB. Essential for identifying DDL operation backlogs and potential schema change bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 572, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL check progress", "group": "TTL", "used_for": "Used for monitoring the progress of TTL (Time To Live) checker operations across TiKV regions. Helps track how quickly TiKV is scanning and processing regions for expired data cleanup, essential for diagnosing TTL performance issues and ensuring timely data expiration.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572009, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Binding Cache Hit / Miss OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of SQL plan binding cache hits versus misses in TiDB. Helps evaluate the effectiveness of plan binding cache configuration and identify whether bindings are being efficiently cached, which is critical for consistent query performance when using SQL plan management features.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_binding_cache_hit_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_binding_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 38, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region across TiKV instances. Helps identify write hotspots, unbalanced region distribution, and diagnose performance issues caused by regions handling excessive write traffic.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Damaged files", "group": "Errors", "used_for": "Used for monitoring corrupted SST files detected in TiKV's RocksDB storage engine and tracking how many damaged files have been deleted. Critical for identifying data integrity issues and storage corruption that could lead to data loss or cluster instability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Job Worker Operations Duration", "group": "DDL", "used_for": "Used for monitoring the 99th percentile latency of DDL worker operations in TiDB, tracking how long it takes to execute schema changes like adding indexes or modifying tables. Critical for diagnosing slow DDL performance issues and identifying which specific DDL operations (by type, action, and result) are causing delays in schema modifications.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, action, result))", "sum(increase(tidb_ddl_worker_operation_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, action, result)/sum(increase(tidb_ddl_worker_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, action, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 573, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL checker actions", "group": "TTL", "used_for": "Used for monitoring the rate of TTL (Time To Live) checker actions performed by TiKV instances, such as scanning and deleting expired data. Helps identify TTL processing activity levels and diagnose issues with automatic data cleanup operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 192, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Backfill Progress In Percentage", "group": "DDL", "used_for": "Used for monitoring the completion percentage of DDL backfill operations in TiDB, including index creation and column modifications. Essential for tracking long-running DDL progress and estimating completion time during schema changes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"modify_column\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 565, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether encryption-at-rest has been successfully initialized on TiKV instances. Essential for confirming that data encryption is properly enabled and operational before storing sensitive data.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Meta Operations Duration 99", "group": "Meta", "used_for": "Used for monitoring the 99th percentile latency of TiDB metadata operations, such as schema changes and DDL operations. Critical for diagnosing slow metadata access issues that can impact query planning and schema modification performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763571994, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "AutoID Client Conn Reset Counter", "group": "Meta", "used_for": "Used for monitoring the rate of AutoID client connection resets in TiDB's meta service. Helps diagnose issues with AutoID allocation service stability, particularly when investigating ID generation failures or connection pool problems between TiDB and the AutoID service.", "component": "TIDB", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "increase(tidb_meta_autoid_client_conn_reset_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 51, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "AutoID Duration", "group": "Meta", "used_for": "Used for monitoring the latency of auto-increment ID allocation operations in TiDB at p80 and p99 percentiles. Critical for diagnosing performance issues when tables with AUTO_INCREMENT columns experience slow inserts or ID generation bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 574, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL checker compact duration", "group": "TTL", "used_for": "Used for monitoring the duration of TTL (Time To Live) checker compaction operations in TiKV, tracking both p99 and p9999 latencies. Essential for diagnosing performance issues with expired data cleanup and identifying when TTL compaction operations are taking unusually long, which could impact storage efficiency and query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 570, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL expire count", "group": "TTL", "used_for": "Used for monitoring the rate of key-value pairs being expired and removed by TiKV's TTL (Time-To-Live) mechanism. Helps track TTL cleanup activity and identify potential issues with expired data accumulation or excessive deletion rates that could impact cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 571, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL expire size", "group": "TTL", "used_for": "Used for monitoring the rate of data size being expired and removed by TiKV's Time-To-Live (TTL) mechanism. Helps track TTL cleanup efficiency and identify potential issues with automatic data expiration, which is critical for managing storage space in time-sensitive data scenarios.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 190, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "OPS", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per second in TiDB, showing the throughput of schema changes like CREATE, ALTER, and DROP operations. Essential for tracking DDL workload intensity and identifying periods of heavy schema modification activity that may impact cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 30, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "Used for monitoring the total size of different Column Families (CF) in TiKV's RocksDB storage engine across the cluster. Essential for tracking storage space usage, identifying data growth patterns, and diagnosing potential storage capacity issues or data imbalance problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 575, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TTL checker poll interval", "group": "TTL", "used_for": "Used for monitoring the polling interval of TiKV's TTL (Time To Live) checker, which determines how frequently the system checks for expired data during garbage collection. Helps diagnose TTL processing efficiency and identify if the checker is running at expected intervals to maintain data lifecycle management.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"tikv_gc_run_interval\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "DDL META OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) worker operations per minute in TiDB, tracking the rate of schema changes like CREATE, ALTER, and DROP operations. Essential for identifying DDL activity patterns, diagnosing schema change bottlenecks, and correlating DDL operations with cluster performance issues.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1708, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "CPU", "group": "", "used_for": "Used for monitoring the CPU utilization rate of TiFlash-Proxy processes across the cluster. Essential for identifying CPU bottlenecks, resource contention, and determining if TiFlash-Proxy nodes are overloaded or require scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tiflash\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 85, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Worker Action OPM", "group": "GC", "used_for": "Used for monitoring the operations per minute of TiDB's garbage collection (GC) worker actions. Helps diagnose GC activity levels and identify potential issues with automatic data cleanup processes that could impact storage space and cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 32, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the number of Raft leader regions actively receiving write operations across TiKV instances. Helps identify write hotspots, uneven leader distribution, and diagnose performance issues caused by concentrated write traffic on specific TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel full events in TiKV, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, thread pool saturation, or resource bottlenecks that prevent TiKV from processing requests efficiently.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 163, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Owner Watcher OPS", "group": "Owner", "used_for": "Used for monitoring the rate of owner watcher operations in TiDB, tracking how frequently nodes watch for ownership changes of distributed components (like DDL owner, statistics owner). Essential for diagnosing issues with leader election, ownership transfer delays, or coordination problems in distributed TiDB operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type, result, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1709, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memory", "group": "", "used_for": "Used for monitoring the resident memory usage of TiFlash-Proxy processes across instances. Essential for tracking memory consumption patterns, detecting memory leaks, and identifying instances approaching memory limits that could impact query performance or stability.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1710, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "IO utilization", "group": "", "used_for": "Used for monitoring the I/O utilization percentage of disks used by TiFlash-Proxy nodes. Essential for identifying I/O bottlenecks and determining if disk saturation is causing performance degradation in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 63, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Backfill Data Rate", "group": "DDL", "used_for": "Used for monitoring the rate of index backfill operations during DDL execution in TiDB. Essential for tracking the throughput of adding indexes to existing data and diagnosing slow DDL performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_add_index_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 34, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Approximate region size", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiKV, showing p99 and p9999 percentiles plus average size. Critical for identifying region size imbalances that can cause hotspots, uneven load distribution, or trigger region split/merge operations affecting cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 33, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Approximate region size", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes across TiKV instances. Helps identify region size imbalances, oversized regions that may need splitting, or issues with region merge/split operations that could impact cluster performance and storage efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 322, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Task Status", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of task statuses in TiDB's distributed execution framework, broken down by task type. Helps identify stuck tasks, track task progression, and diagnose issues in distributed query execution or DDL operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (status, task_type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572001, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Add Index Scan Rate", "group": "DDL", "used_for": "Used for monitoring the row scanning throughput (rows per second) during ADD INDEX operations in TiDB at different percentiles (p50, p95, p999). Essential for evaluating index creation performance, identifying scanning bottlenecks, and estimating completion time for large table index additions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763573003, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Add Index Backfill Import Speed", "group": "DDL", "used_for": "Used for monitoring the data import speed (bytes per second) during index backfill operations in TiDB's distributed execution framework. Essential for tracking DDL performance when adding indexes to large tables and identifying slow import phases that may impact overall index creation time.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}[2m])) by(job_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763573002, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Retryable Error", "group": "DDL", "used_for": "Used for monitoring the rate of retryable errors encountered during DDL (Data Definition Language) operations in TiDB. Helps identify transient issues like schema conflicts or resource contention that cause DDL statements to retry, which may indicate cluster instability or concurrent DDL execution problems.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_ddl_retryable_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1714, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region", "group": "", "used_for": "Used for monitoring the total number of Raft regions managed by each TiFlash-Proxy instance. Essential for tracking region distribution across TiFlash nodes and diagnosing data skew or unbalanced region allocation issues that could impact query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 326, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Completed/Total Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for monitoring the progress of distributed task execution by tracking completed subtasks versus total subtasks for each task type and ID. Essential for diagnosing stuck or slow distributed operations like DDL, import, or analyze jobs, and identifying whether tasks are progressing normally or encountering failures.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"succeed|failed|canceled|reverted|revert_failed\"}) by (task_id, task_type)", "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (task_id, task_type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 158, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Delete Range Failure OPM", "group": "GC", "used_for": "Used for monitoring the failure rate of TiDB's delete range operations during garbage collection, measured in operations per minute. Critical for identifying issues with physical data deletion in TiKV, which can lead to disk space not being reclaimed properly and potential storage problems.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4106, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Uptime", "group": "", "used_for": "Used for monitoring the continuous running time of TiFlash-Proxy processes since their last restart. Helps identify unexpected process restarts, assess cluster stability, and determine if performance issues correlate with recent process lifecycle events.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - tiflash_proxy_process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 329, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pending Subtask Count", "group": "Dist Execute Framework", "used_for": "Used for monitoring the number of pending subtasks in TiDB's distributed execution framework, grouped by executor ID. Helps identify task scheduling bottlenecks, executor overload, or stalled distributed operations when pending counts remain high or grow continuously.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending\"}) by (exec_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 90, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Too Many Locks Error OPM", "group": "GC", "used_for": "Used for monitoring the rate of \"too many locks\" errors during garbage collection operations per minute. Indicates when GC processes encounter regions with excessive locks, which can signal transaction conflicts, long-running transactions blocking cleanup, or potential GC performance degradation requiring investigation.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 330, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "SubTask Running Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the execution duration of currently running subtasks in TiDB's distributed task execution framework. Helps identify long-running subtasks that may indicate performance bottlenecks or stuck operations in distributed query processing.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"running\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 35, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Huge memtable size", "group": "Server", "used_for": "Used for monitoring the size distribution of oversized memtables in TiKV regions, tracking both p99 and p9999 percentiles. Critical for diagnosing memory pressure issues and identifying regions with abnormally large in-memory write buffers that could impact write performance and memory stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_region_huge_mem_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region across TiKV instances. Helps identify write hotspots and uneven data distribution patterns that could impact cluster performance and load balancing.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "GC Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection failures per minute in TiDB, categorized by failure type. Critical for identifying GC issues that can lead to storage bloat, performance degradation, and potential cluster instability when old MVCC versions cannot be cleaned up properly.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 37, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the total bytes written to regions across TiKV instances. Helps identify write-heavy regions, diagnose unbalanced write distribution, and detect potential hotspot issues that could impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 89, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Action Result OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) action results per minute in TiDB, categorized by operation type. Essential for tracking GC activity patterns and diagnosing issues with MVCC version cleanup or storage reclamation efficiency.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4540, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Duration", "group": "Raft Engine", "used_for": "Used for monitoring write operation latency in TiFlash-Proxy's Raft Engine, tracking both p99 and p9999 percentiles plus average duration. Critical for diagnosing TiFlash replication performance issues and identifying write bottlenecks that could impact data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tiflash_proxy_raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 182, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Resolve Locks Range Tasks Push Task Duration 95", "group": "GC", "used_for": "Used for monitoring the 95th percentile duration of pushing resolve locks range tasks to TiKV during garbage collection. Critical for diagnosing GC performance issues and identifying delays in lock resolution that could impact transaction cleanup and storage reclamation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"resolve-locks-runner.*\"}[2m])) by (le, instance, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 39, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the number of keys written to regions in TiKV instances. Helps identify write hotspots, uneven data distribution, and regions experiencing heavy write traffic that may need splitting or load balancing.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4554, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Flow", "group": "Raft Engine", "used_for": "Used for monitoring the data flow rates in TiFlash-Proxy's Raft Engine, including write throughput and background rewrite activity. Essential for diagnosing storage I/O performance issues and understanding compaction overhead in TiFlash's replication layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tiflash_proxy_raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tiflash_proxy_raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 321, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Ongoing User Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of active user transactions in TiDB. Critical for identifying long-running transactions that may block garbage collection (GC) and cause MVCC version accumulation, leading to performance degradation and storage bloat.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 328, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Slots Usage", "group": "Dist Execute Framework", "used_for": "Used for monitoring the current slot usage versus maximum available slots in TiDB's distributed task execution framework. Helps identify resource saturation and capacity planning needs when distributed tasks are consuming all available worker slots, which could lead to task queuing or execution delays.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_used_slots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 87, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Config", "group": "GC", "used_for": "Used for monitoring TiDB's garbage collection (GC) configuration parameters across the cluster. Helps verify GC settings are properly configured and identify any configuration inconsistencies that could affect data cleanup and storage efficiency.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 181, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Resolve Locks Range Tasks Status", "group": "GC", "used_for": "Used for monitoring the status and results of resolve locks range tasks in TiDB's garbage collection process. Helps diagnose lock resolution efficiency and identify issues when cleaning up stale locks during GC operations, which is critical for maintaining transaction consistency and preventing lock accumulation.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"resolve-locks-runner.*\"}) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 331, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Subtask Pending Duration", "group": "Dist Execute Framework", "used_for": "Used for monitoring the time distributed task subtasks spend in pending state before execution in TiDB's distributed execution framework. Critical for identifying task scheduling bottlenecks, resource contention, or executor availability issues that delay subtask processing.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_subtask_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"pending\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 36, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Huge l0 table size", "group": "Server", "used_for": "Used for monitoring the size distribution of large L0 SST tables in TiKV's RocksDB engine, tracking both p99 and p9999 percentiles. Critical for identifying write amplification issues and compaction bottlenecks that occur when L0 tables grow excessively large, which can severely impact write performance and trigger read/write stalls.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n kv_engine_region_huge_l0_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 42, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Written key size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total size of keys written through the Raft apply process in TiKV. Helps diagnose write workload characteristics and identify issues related to large key writes that may impact storage performance and memory usage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Hibernate Peers", "group": "Server", "used_for": "Used for monitoring the number of hibernated Raft peers in TiKV instances. Helps identify regions that have entered hibernation mode due to inactivity, which is useful for understanding cluster resource optimization and detecting potential issues with region state management.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4560, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memory trace", "group": "", "used_for": "Used for tracking detailed memory usage breakdown in TiFlash-Proxy components, specifically monitoring raftstore and raft engine memory consumption. Essential for diagnosing memory-related issues in TiFlash-Proxy and identifying which component is consuming excessive memory during troubleshooting.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tiflash_proxy_tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "((\n tiflash_proxy_raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Inaccuracy Rate", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the inaccuracy rate of table statistics in TiDB at different percentiles (p50, p90, p99). Critical for identifying when statistics are outdated or inaccurate, which can lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 4538, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Operation", "group": "Raft Engine", "used_for": "Used for monitoring the rate of I/O operations in TiFlash-Proxy's Raft Engine, including write apply, entry reads, and message reads. Essential for diagnosing Raft log replication performance and identifying I/O bottlenecks in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tiflash_proxy_raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tiflash_proxy_raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "sum(rate(\n tiflash_proxy_raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 538, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Failure Reason", "group": "Backup Log", "used_for": "Used for monitoring the failure reasons of region checkpoint operations during TiDB log backup. Helps diagnose why backup log regions are failing by categorizing error types, enabling quick identification of backup reliability issues and their root causes.", "component": "TIKV", "phenomenon": "error_rate", "object": "region", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 71, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Pseudo Estimation OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of pseudo statistics usage in query optimization across TiDB instances. Helps identify when the optimizer lacks accurate table statistics, which can lead to suboptimal query plans and performance degradation requiring ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 332, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Uncompleted Subtask Distribution On TiDB Nodes", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of incomplete subtasks across TiDB nodes in the distributed execution framework. Helps identify workload imbalance and potential bottlenecks when subtasks are unevenly distributed or stuck in pending/running/failed states on specific nodes.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending|running|failed|canceled|paused\"}) by (exec_id)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 233, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Healthy Distribution", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the health distribution of table statistics in TiDB, categorized by statistics type. Essential for identifying stale or outdated statistics that could lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_statistics_stats_healthy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 41, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Clear overlap region duration", "group": "Server", "used_for": "Used for monitoring the latency of clearing overlapping regions in TiKV's Raftstore, tracking both p99 and p9999 percentiles. Critical for diagnosing region merge/split performance issues and identifying when region overlap cleanup operations are experiencing delays that could impact cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "region", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 309, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Ongoing Internal Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of ongoing internal transactions in TiDB, such as those used by GC (Garbage Collection) and statistics operations. Critical for identifying when internal background transactions are taking too long, which can impact cluster performance and cause resource contention with user queries.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"internal\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 47, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Resolve address duration", "group": "Server", "used_for": "Used for monitoring the latency of DNS/address resolution operations in TiKV servers. Critical for diagnosing network connectivity issues and identifying when slow DNS lookups are impacting cluster communication and overall performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4550, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Written", "group": "Raft Engine", "used_for": "Used for monitoring the write size distribution in TiFlash-Proxy's Raft Engine, tracking p99 and p9999 percentiles plus average bytes written per operation. Essential for diagnosing write amplification issues and identifying abnormal write patterns that could impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tiflash_proxy_raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tiflash_proxy_raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by () " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 203, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "No Available Connection Counter", "group": "Batch Client", "used_for": "Used for monitoring the rate of connection unavailability in TiDB's batch client when communicating with TiKV. Helps diagnose connection pool exhaustion or network connectivity issues that may cause query failures or performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 46, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft Entry Cache Evicts", "group": "Server", "used_for": "Used for monitoring the rate at which Raft log entries are evicted from TiKV's entry cache. Helps diagnose memory pressure issues and cache efficiency problems when the cache cannot hold enough recent Raft entries, potentially impacting replication performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 47, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Auto/Manual Analyze Queries Per Minute", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the frequency of automatic and manual ANALYZE operations per minute in TiDB. Helps identify statistics collection patterns, detect excessive auto-analyze activity that may impact cluster performance, and verify manual statistics maintenance schedules are executing as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_manual_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 43, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Written value size for every put of apply worker", "group": "Server", "used_for": "Used for monitoring the distribution of value sizes written during Raft apply operations in TiKV. Helps diagnose write amplification issues and identify whether large value writes are impacting apply worker performance and storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4548, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Other Durations (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency of miscellaneous Raft Engine operations in TiFlash-Proxy, including reading entries, reading messages, and purging operations. Essential for identifying tail latency issues in Raft Engine's auxiliary operations that could impact TiFlash replication performance and data consistency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 86, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "GC Duration By Stage", "group": "GC", "used_for": "Used for monitoring the maximum duration of different garbage collection stages in TiDB's distributed transaction system. Critical for diagnosing GC performance bottlenecks and identifying which specific stage (resolve locks, delete ranges, etc.) is causing slow cleanup of obsolete MVCC versions.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (stage, le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 46, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Auto/Manual Analyze Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the duration of automatic and manual ANALYZE operations at the 80th and 95th percentiles. Essential for identifying slow statistics collection that could delay query optimization or impact cluster performance during maintenance windows.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"AnalyzeTable\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"AnalyzeTable\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 4556, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Duration Breakdown (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency breakdown of TiFlash-Proxy's Raft Engine write operations across three stages: preprocessing, leader writing, and applying. Essential for diagnosing write performance bottlenecks in TiFlash's Raft log persistence layer by identifying which specific write phase is causing latency spikes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 204, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Wait Connection Establish Duration", "group": "Batch Client", "used_for": "Used for monitoring the 99.99th percentile latency when TiDB batch clients wait to establish connections to TiKV. Critical for diagnosing connection pool exhaustion, network issues, or TiKV overload that causes extreme delays in establishing new connections.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4546, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "File Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of files managed by TiFlash-Proxy's Raft Engine, including log files, swap files, and recycled files. Essential for tracking storage file management health and identifying potential issues with file accumulation or cleanup processes that could impact disk space usage.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tiflash_proxy_raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "avg((\n tiflash_proxy_raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", "avg((\n tiflash_proxy_raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 45, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Memory trace", "group": "Server", "used_for": "Used for tracking detailed memory usage breakdown in TiKV instances, specifically monitoring raftstore threads and Raft Engine memory consumption. Essential for diagnosing memory pressure issues and identifying which TiKV components are consuming excessive memory resources.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) ", "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Thread Pool Schedule Wait Duration$optional_quantile", "group": "Server", "used_for": "Used for monitoring the time tasks spend waiting in TiKV's thread pool queue before being scheduled for execution. Critical for identifying thread pool saturation and resource contention issues that cause request processing delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 93, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Update Stats OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of statistics update operations in TiDB, broken down by update type. Essential for tracking the health of automatic statistics collection and identifying potential bottlenecks in the statistics maintenance process that could affect query optimization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 228, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Batch Receive Average Duration", "group": "Batch Client", "used_for": "Used for monitoring the average and P99 latency of batch receive operations between TiDB and TiKV clients. Critical for diagnosing network communication delays and identifying performance bottlenecks in the batch RPC layer when queries experience slow response times.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 305, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Ignore Event Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the rate of SQL events ignored by TopSQL feature per minute, categorized by ignore type. Helps identify when TopSQL is dropping events due to resource constraints or configuration limits, which may indicate the need to adjust TopSQL settings or investigate high query volumes.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_ignored_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 180, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Batch Client Unavailable Duration 95", "group": "Batch Client", "used_for": "Used for monitoring the 95th percentile duration when TiDB's batch client cannot send requests to TiKV due to unavailability. Critical for diagnosing connection issues, network problems, or TiKV overload situations that prevent TiDB from batching and sending requests efficiently.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4552, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL Duration Breakdown (999%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency breakdown of TiFlash-Proxy's Raft Engine WAL (Write-Ahead Log) operations, including leader write, log sync, and log allocation phases. Critical for diagnosing tail latency issues in TiFlash's Raft log persistence and identifying which specific WAL operation stage is causing write performance bottlenecks.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4542, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Entry Count", "group": "Raft Engine", "used_for": "Used for monitoring the average count of Raft log entries in TiFlash-Proxy's Raft Engine, categorized by entry type. Helps diagnose Raft log accumulation issues, replication lag, and potential memory pressure from unbounded log growth in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((\n tiflash_proxy_raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4544, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Compression Ratio", "group": "Raft Engine", "used_for": "Used for monitoring the compression ratio achieved when writing data to TiFlash's Raft Engine, showing p99, p9999, and average compression effectiveness. Essential for evaluating storage efficiency and identifying compression performance issues that could impact disk space usage and I/O overhead in TiFlash proxy instances.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tiflash_proxy_raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 239, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "99 Report Duration", "group": "TopSQL", "used_for": "Used for monitoring the 99th percentile latency of TopSQL report generation and submission in TiDB and TiKV. Critical for diagnosing performance issues in the TopSQL feature itself, helping identify when report collection overhead becomes excessive or when there are delays in transmitting query statistics to the monitoring system.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"ok\"}[5m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[5m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 229, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Sync Load QPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the throughput of synchronous statistics loading operations in TiDB, including total load requests, timeout occurrences, and deduplicated requests. Essential for diagnosing query optimizer performance issues related to statistics availability and identifying when statistics loading becomes a bottleneck or experiences timeouts.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_dedup_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 49, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Average Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the average time tasks spend waiting in TiKV's thread pool queues before execution. Critical for diagnosing thread pool saturation and scheduling bottlenecks that cause request processing delays, especially when TiKV appears slow despite low CPU usage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )", "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2741, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiFlash-Proxy instances, categorized by error type. Essential for detecting severe operational issues that require immediate attention, such as data corruption, storage failures, or system-level problems that could impact TiFlash's analytical query processing.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 50, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Disk IO time per second", "group": "Server", "used_for": "Used for monitoring the total time spent on disk I/O operations per second in TiKV's RocksDB storage engine, including both storage and coprocessor read operations. Critical for diagnosing disk performance bottlenecks and identifying when slow disk I/O is impacting query and storage performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1718, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiFlash Proxy's Raftstore engine. Critical for identifying storage layer errors and replication issues that could impact TiFlash's data consistency and query performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 54, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV instances, excluding garbage collection operations. Critical for identifying communication issues between TiKV nodes or with TiDB/PD components that could impact cluster stability and query performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC message duration$optional_quantile", "group": "gRPC", "used_for": "Used for monitoring the latency of gRPC message processing in TiKV at configurable percentiles. Essential for diagnosing communication delays between TiDB/TiKV components and identifying network or RPC-level performance bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 242, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Report Data Count Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the volume of TopSQL data being reported per minute from TiDB instances, broken down by data type. Helps diagnose TopSQL collection overhead and identify potential issues with SQL performance data reporting or collection bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_report_data_total_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type, job)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 51, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Disk IO bytes per second", "group": "Server", "used_for": "Used for monitoring the rate of disk I/O operations in bytes per second across TiKV instances, including both storage and coprocessor read activities from RocksDB. Essential for identifying disk bottlenecks, diagnosing I/O saturation issues, and correlating storage performance with query latency problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 245, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiKV Stat Task OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of resource metering statistics tasks processed by TiKV instances. Helps diagnose TopSQL feature performance and identify potential bottlenecks in query resource tracking and statistics collection.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tikv_resource_metering_stat_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 53, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages processed by TiKV instances, broken down by message type (excluding garbage collection). Essential for diagnosing communication bottlenecks between TiDB/TiKV nodes and identifying which gRPC operations are experiencing high traffic or potential issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 230, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Sync Load Latency P9999", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99.99th percentile latency of synchronous statistics loading operations in TiDB, including both overall sync load time and the time spent reading statistics from storage. Critical for diagnosing extreme tail latency issues in query optimization when TiDB needs to load table statistics on-demand, which can impact query planning performance during cold starts or after statistics updates.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 243, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Total Report Count", "group": "TopSQL", "used_for": "Used for monitoring the volume and success rate of TopSQL report submissions from TiDB instances to the diagnostic system. Essential for verifying that query performance data is being properly collected and transmitted, helping diagnose issues when TopSQL metrics are missing or incomplete in PingCAP Clinic diagnostics.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type, result)", "sum(increase(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type, result)", "sum(tikv_resource_metering_report_data_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"sent\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 297, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TiDB CPU Usage", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiDB instances in relation to their maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks during TTL (Time To Live) operations and determining if TiDB instances need vertical scaling or workload optimization.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 241, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "CPU Profiling OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of CPU profiling operations in TiDB instances. Helps track TopSQL feature activity and diagnose overhead from continuous profiling, useful when investigating performance impact of query monitoring itself.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_server_cpu_profile_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 235, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Cache OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per second on TiDB's statistics cache, broken down by operation type (hits, misses, evictions). Essential for diagnosing statistics cache efficiency and identifying when cache misses may be causing suboptimal query plan generation or performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_stats_cache_op{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 237, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Historical Stats OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the operations per minute (OPM) of historical statistics generation and dumping in TiDB. Helps diagnose issues with statistics collection and persistence by tracking success/failure rates of stats operations, which is critical for maintaining optimal query plan selection.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[2m])) by (result)", "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 18, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiFlash-Proxy nodes, categorized by failure type and instance. Critical for identifying communication issues between TiFlash-Proxy and PD/TiKV that could indicate network problems, node failures, or cluster instability affecting data replication and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1719, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring error rates in TiFlash-Proxy's scheduler operations, specifically tracking snapshot and prepare_write stage failures. Critical for diagnosing TiFlash write path issues and identifying nodes experiencing scheduling problems that could impact data replication or query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1584, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiFlash-Proxy server overload conditions, including scheduler busy errors, channel saturation, and coprocessor request rejections due to resource exhaustion. Critical for diagnosing when TiFlash-Proxy instances are overwhelmed by request volume or experiencing backpressure, indicating the need for capacity scaling or workload optimization.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 236, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Plan Replayer Task OPM", "group": "Statistics & Plan Management", "used_for": "Used for monitoring Plan Replayer task execution rates in TiDB, tracking both dump and capture operations per minute along with registered task counts. Essential for diagnosing issues with SQL plan reproduction and troubleshooting query performance problems, as Plan Replayer helps capture execution plan context for offline analysis.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)", "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"capture\"}[2m])) by (result)", "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 234, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Cache Cost", "group": "Statistics & Plan Management", "used_for": "Used for monitoring TiDB's statistics cache memory usage, tracking both current tracked memory and total capacity. Essential for diagnosing memory pressure issues in the stats cache and determining if cache size needs adjustment to prevent statistics eviction that could lead to suboptimal query plans.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"track\"}", "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"capacity\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 56, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Average gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC message processing in TiKV instances. Essential for diagnosing communication delays between TiDB/TiKV components and identifying network or RPC-level performance bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572006, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Loading Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile duration of statistics delta loading operations in TiDB. Critical for diagnosing performance issues related to table statistics updates, which can impact query optimization and execution plan generation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_load_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1722, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region drops in TiFlash-Proxy instances. Helps identify leader transfer issues, network instability, or node failures that cause unexpected leader region losses, which can impact query performance and data availability.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1720, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiFlash-Proxy, broken down by instance and error reason. Essential for diagnosing data processing failures and identifying specific error patterns that may indicate issues with query execution, data consistency, or communication problems between TiFlash and TiKV components.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 279, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL QPS By Type", "group": "TTL", "used_for": "Used for monitoring the queries per second (QPS) of TTL (Time To Live) operations in TiDB, broken down by SQL type and execution result. Helps identify TTL workload patterns, track the volume of automatic data cleanup operations, and diagnose issues when TTL delete operations are failing or experiencing unexpected throughput changes.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 303, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Delete Rows Per Hour", "group": "TTL", "used_for": "Used for tracking the hourly rate of rows successfully deleted by TiDB's Time-To-Live (TTL) feature. Helps monitor TTL cleanup efficiency and identify potential issues when expired data isn't being purged as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the batch size of gRPC requests and responses in TiKV, showing both percentile distributions and average values. Essential for diagnosing gRPC batching efficiency and identifying whether request/response batching is working optimally to reduce network overhead.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572009, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Binding Cache Hit / Miss OPS", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the rate of SQL plan binding cache hits versus misses in TiDB. Helps evaluate the effectiveness of plan binding cache configuration and identify when bindings aren't being cached efficiently, which could impact query optimization performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_binding_cache_hit_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_binding_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1723, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions in TiFlash-Proxy that are missing their leaders. Critical for diagnosing replication issues and cluster instability, as missing leaders indicate regions that cannot serve read/write requests until leadership is re-established.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 302, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows inserted per second by TiDB's Time-To-Live (TTL) feature. Helps track TTL background job activity and diagnose performance issues when TTL operations are inserting data back into tables or temporary storage.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572008, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Binding Cache Memory Usage", "group": "Statistics & Plan Management", "used_for": "Used for monitoring memory consumption of TiDB's SQL binding cache, which stores execution plan bindings. Helps identify memory pressure from plan management features and diagnose issues when binding cache approaches its configured limit, potentially causing binding evictions or memory-related performance problems.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_mem_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_server_binding_cache_mem_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1721, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by instance and message type. Essential for identifying communication issues between TiFlash and TiKV, helping diagnose network problems, timeout errors, or service unavailability that could impact query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 298, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the hourly rate of rows inserted by TTL (Time To Live) background jobs in TiDB. Helps track TTL job activity and identify unusual spikes or drops in TTL-managed data insertion, which may indicate configuration issues or workload changes affecting expired data handling.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1h]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 287, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Processed Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows processed by TiDB's Time-To-Live (TTL) feature, broken down by SQL operation type and result status. Essential for tracking TTL job performance, identifying processing bottlenecks, and ensuring expired data is being cleaned up at the expected rate.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 284, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Scan Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution of TTL (Time To Live) scan queries at various percentiles (p50, p80, p90, p99). Essential for diagnosing performance issues with TTL background jobs that automatically delete expired data, helping identify when TTL operations are impacting cluster performance or taking longer than expected.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))\n", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 58, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the batch size of Raft messages sent between TiKV nodes via gRPC. Helps diagnose network efficiency and Raft communication patterns, where abnormally small batches may indicate network issues or configuration problems, while large batches suggest healthy message aggregation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "optional_quantile", "tidb_cluster" ], "promql": [ "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel saturation events in TiFlash-Proxy, indicating when internal message queues become full. Critical for diagnosing backpressure issues, communication bottlenecks between TiFlash and TiKV, and potential performance degradation due to overwhelmed processing capacity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 163, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Owner Watcher OPS", "group": "Owner", "used_for": "Used for monitoring the rate of owner watcher operations in TiDB's distributed coordination system. Helps diagnose issues with DDL owner election, schema synchronization, and cluster coordination by tracking watcher operation throughput and success/failure rates across instances.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result, instance)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 59, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC request sources QPS", "group": "gRPC", "used_for": "Used for monitoring the queries per second of gRPC requests broken down by their source components in TiKV. Helps identify which components (TiDB, PD, or other TiKV nodes) are generating the most gRPC traffic and diagnose communication bottlenecks or abnormal request patterns between cluster components.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 60, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC request sources duration", "group": "gRPC", "used_for": "Used for monitoring the duration of gRPC requests grouped by their source in TiKV. Helps identify which request sources (e.g., TiDB nodes, PD, or other TiKV instances) are experiencing high latency, enabling targeted troubleshooting of inter-component communication issues.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC resource group QPS", "group": "gRPC", "used_for": "Used for monitoring the queries per second (QPS) handled by each resource group in TiKV's gRPC layer. Essential for understanding workload distribution across resource groups and identifying which groups are consuming the most gRPC resources, helping diagnose resource contention and quota enforcement issues.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "New ETCD Session Duration 95", "group": "Owner", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile latency when TiDB establishes new ETCD sessions for owner election and coordination. Critical for diagnosing issues with cluster metadata operations, DDL job scheduling, and detecting ETCD connectivity or performance problems that could impact cluster coordination.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, result))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 285, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Delete Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution (p50, p80, p90, p99) of TTL (Time To Live) delete queries in TiDB. Essential for diagnosing performance issues with automatic data expiration operations and identifying when TTL background deletion tasks are experiencing slowdowns that could impact cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1705, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the total storage size of each TiFlash-Proxy instance's RocksDB engine. Essential for tracking disk space usage, capacity planning, and identifying instances approaching storage limits that may require scaling or data cleanup.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 282, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Delete Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) delete worker operations in TiDB. Helps identify performance bottlenecks in the TTL deletion process by showing which phase (e.g., scanning, deleting, waiting) consumes the most time.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"delete_worker\"}[2m])) by (phase)\n" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 33, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total storage size of different column families (CFs) in TiFlash-Proxy's RocksDB engine. Helps track storage space consumption patterns and identify potential storage bloat or data distribution issues across lock, default, and write column families.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 52, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Meta Operations Duration 99", "group": "Meta", "used_for": "Used for monitoring the 99th percentile latency of TiDB metadata operations (such as schema changes, table creation, and DDL operations). Critical for diagnosing slow DDL execution and identifying metadata-related performance bottlenecks that could impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572010, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Number Of Bindings In Cache", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the number of SQL plan bindings currently cached in TiDB's memory. Helps track plan binding usage and diagnose issues when execution plans aren't being enforced as expected due to cache eviction or binding management problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_binding_cache_num_bindings{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 65, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the total rate of storage commands processed by TiKV instances. Helps identify overall storage operation throughput and detect sudden changes in command volume that may indicate application behavior changes or potential issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the rate of key writes across Raft leader regions on each TiFlash node. Helps identify write hotspots, uneven write distribution, and diagnose performance issues related to leader region write activity in TiFlash's proxy layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 281, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Job Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) jobs across different statuses in TiDB. Helps identify job execution patterns, detect stuck or failed TTL jobs, and assess the overall health of automatic data expiration processes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 62, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC Compression Status for Msg", "group": "gRPC", "used_for": "Used for monitoring the compression types applied to gRPC messages in TiKV, showing which compression algorithms (if any) are being used for inter-component communication. Helps diagnose network bandwidth usage and verify compression configuration effectiveness when troubleshooting communication overhead between TiKV nodes or with TiDB/PD components.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_grpc_compression_type_for_msg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 276, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Scan Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) scan workers in TiDB. Helps identify which phase (scanning, deleting, or waiting) is consuming the most time, enabling diagnosis of TTL job performance bottlenecks and optimization opportunities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=\"scan_worker\"}[2m])) by (phase)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3638, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiFlash-Proxy's Raft store. Helps identify region size imbalances and diagnose issues related to region splitting, merging, or uneven data distribution that could impact I/O performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1481, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the distribution of Region sizes in TiFlash-Proxy's Raft store, showing p95, p99 percentiles and average values. Essential for identifying oversized Regions that could impact I/O performance, replication efficiency, and detecting potential Region split issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 63, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC Compression Status for Raft Replication", "group": "gRPC", "used_for": "Used for monitoring the compression algorithm status applied to gRPC messages during Raft log replication between TiKV nodes. Helps identify whether compression is enabled and which compression type is being used, which is critical for diagnosing network bandwidth utilization and replication performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "((\n tikv_grpc_compression_type_for_raft\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 66, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the error rate of asynchronous storage requests in TiKV's storage engine. Critical for detecting storage layer failures, timeout issues, or resource exhaustion that could impact data read/write operations and overall cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region in TiFlash-Proxy instances. Helps identify uneven write distribution across regions and diagnose I/O performance issues when certain instances show significantly higher write volumes than others.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 50, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "AutoID QPS", "group": "Meta", "used_for": "Used for monitoring the rate of AutoID allocation operations per second in TiDB. Helps identify workload patterns involving auto-increment columns and diagnose performance issues related to ID generation, especially during high-volume insert operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 294, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Task Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) tasks across different statuses in TiDB. Helps track task execution progress and identify stuck or failed TTL cleanup operations that may impact data retention policies.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 67, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying delays in data persistence to the underlying storage layer.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571994, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "AutoID Client Conn Reset Counter", "group": "Meta", "used_for": "**Used For:**\n\nUsed for tracking the rate of AutoID client connection resets in TiDB's meta service. Helps diagnose issues with AutoID allocation stability and identify potential network problems or service disruptions affecting automatic ID generation.", "component": "TIDB", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_meta_autoid_client_conn_reset_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 293, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "EMA CPU Usage", "group": "Resource Manager", "used_for": "Used for monitoring the Exponential Moving Average (EMA) of CPU usage by TiDB's Resource Manager across different resource groups or keyspaces. Essential for tracking smoothed CPU consumption trends and identifying resource allocation issues or imbalances in multi-tenant environments.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 51, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "AutoID Duration", "group": "Meta", "used_for": "Used for monitoring the latency of TiDB's auto-increment ID allocation operations at p80 and p99 percentiles. Critical for diagnosing performance issues when applications experience delays in obtaining auto-increment IDs, particularly during high-concurrency insert workloads or when the auto-ID cache needs replenishment from PD.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 69, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine. Critical for diagnosing performance issues with snapshot reads and identifying bottlenecks in async request processing that may impact query response times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 85, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Worker Action OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) worker operations per minute in TiDB, broken down by action type. Essential for tracking GC activity patterns and diagnosing issues with distributed transaction cleanup or storage reclamation performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 3636, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the rate of bytes written to regions in TiFlash-Proxy per second. Helps identify write-heavy workloads, diagnose I/O bottlenecks, and detect abnormal write patterns that may impact TiFlash storage performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 57, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region in TiFlash-Proxy. Helps identify write hotspots and uneven data distribution across regions, which can indicate workload imbalances or the need for region splitting.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "GC Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) failures per minute in TiDB, categorized by failure type. Critical for identifying GC process issues that can lead to storage bloat, performance degradation, and potential cluster instability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 300, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Table Count By TTL Schedule Delay", "group": "TTL", "used_for": "Used for monitoring the number of tables grouped by their TTL (Time To Live) schedule delay watermark in TiDB. Helps identify tables experiencing scheduling delays in TTL cleanup operations, which is critical for diagnosing data retention issues and ensuring timely deletion of expired data.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "max(tidb_server_ttl_watermark_delay{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", type=\"schedule\"}) by (type, name)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 158, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Delete Range Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of failures per minute when TiDB attempts to delete data ranges during garbage collection. Critical for identifying issues with the unsafe destroy range operation that could lead to storage space not being reclaimed properly or GC process failures.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 304, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Insert/Delete Rows By Day", "group": "TTL", "used_for": "Used for monitoring the daily volume of TTL (Time To Live) operations, showing both rows inserted and expired rows deleted. Essential for tracking TTL job effectiveness, identifying data lifecycle patterns, and diagnosing issues when expired data isn't being cleaned up as expected.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 313, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "TTL Event Count Per Minute", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) related events occurring per minute in TiDB, including TTL operations, timer events, and runtime activities. Essential for tracking TTL feature activity, diagnosing automatic data cleanup performance, and identifying issues with expired row deletion processes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=\"hook.tidb.ttl\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", scope=~\"runtime\\\\.ttl.*\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3637, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the rate of keys written per region in TiFlash-Proxy's storage engine. Helps identify write-intensive regions and diagnose uneven write distribution or hotspot issues that could impact TiFlash data synchronization performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3720, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of requests received by TiFlash-Proxy server, showing both average and p99 percentile values by request type. Essential for identifying request batching efficiency and detecting anomalies where requests aren't being properly batched, which could indicate client-side issues or network problems affecting TiFlash query performance.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 87, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Config", "group": "GC", "used_for": "Used for monitoring TiDB's garbage collection (GC) configuration parameters across the cluster. Helps verify GC settings are properly configured and identify configuration inconsistencies that could impact storage reclamation and cluster performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 86, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Duration 99", "group": "GC", "used_for": "Used for monitoring the 99th percentile latency of TiDB's garbage collection (GC) operations. Essential for identifying GC performance degradation that could impact transaction processing and cluster stability.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 89, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Action Result OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) action results per minute in TiDB, categorized by operation type. Helps diagnose GC efficiency and identify issues with data cleanup operations, such as failed delete ranges or resolve locks operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 71, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot read operations in TiKV's storage engine when data is read purely from local storage without cross-region access. Critical for diagnosing local read performance issues and identifying bottlenecks in snapshot isolation reads that don't require remote data fetching.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiKV's storage engine, tracking both P99 and P9999 percentiles. Critical for diagnosing write performance bottlenecks and identifying tail latency issues in the storage layer that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 289, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "GOGC", "group": "Resource Manager", "used_for": "**Used For:**\n\nUsed for monitoring the GOGC (Go Garbage Collection) parameter value in TiDB instances, which controls the aggressiveness of Go's garbage collector. Essential for diagnosing memory management issues and understanding GC behavior when investigating memory pressure or performance degradation related to garbage collection overhead.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3718, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch ratio", "group": "Server", "used_for": "Used for monitoring the batch efficiency of requests processed by TiFlash-Proxy, showing both average and P99 batch ratios by request type. Essential for identifying request batching inefficiencies that could impact TiFlash I/O performance and throughput optimization.", "component": "TIFLASH", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 310, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Closest Replica Hit Count Per Second", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor requests served by the closest replica (follower read) per second in TiDB. Helps evaluate the effectiveness of follower read optimization and diagnose whether read requests are being properly distributed to nearby replicas to reduce cross-AZ latency.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_closest_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 107, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by message type. Essential for identifying communication issues between TiFlash and other TiDB components, helping diagnose network problems or service degradation.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 90, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Too Many Locks Error OPM", "group": "GC", "used_for": "Used for monitoring the rate of \"too many locks\" errors encountered during garbage collection operations in TiDB. Helps diagnose GC failures caused by excessive locks in regions, which can indicate long-running transactions blocking cleanup or potential transaction conflicts requiring investigation.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 324, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Coprocessor Response Size Per Instance And TiKV", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor response data size from TiKV stores to each TiDB instance, particularly in follower read scenarios. Helps identify network bandwidth consumption patterns and diagnose issues where specific TiKV stores are sending disproportionate amounts of data, which may indicate data skew or inefficient query patterns.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 73, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the time spent waiting for read index proposals in TiKV's storage engine. Critical for diagnosing read latency issues caused by Raft read index delays, particularly when investigating slow follower reads or leader lease problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 98, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message handling in TiFlash-Proxy. Critical for identifying communication bottlenecks between TiFlash and TiKV, helping diagnose slow data synchronization or query execution issues caused by inter-component network delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 321, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Ongoing User Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of active user transactions in TiDB. Critical for identifying long-running transactions that may block garbage collection, cause lock contention, or indicate application-level issues requiring immediate attention.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 95, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC message types processed by TiFlash-Proxy, excluding garbage collection messages. Helps identify communication patterns between TiFlash and TiKV, and diagnose issues with specific gRPC message types when experiencing replication delays or data synchronization problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine, tracking both P99 and P9999 percentiles. Critical for diagnosing slow snapshot read performance issues and identifying tail latency problems that could impact transaction consistency checks and MVCC operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot read operations in TiKV that are served entirely from local storage without requiring Raft replication. Critical for diagnosing performance issues with follower reads and local read optimizations, particularly when investigating tail latency (p99, p9999) that may indicate storage engine bottlenecks or resource contention.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2532, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Average gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC messages between TiFlash-Proxy and other components, broken down by message type. Essential for diagnosing communication bottlenecks and identifying which specific gRPC message types are experiencing elevated latency in TiFlash operations.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 316, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Total Encode/Deliver/Import-kv Speed", "group": "Import Into", "used_for": "Used for monitoring the throughput of IMPORT INTO operations across three stages: data encoding (restored), delivery to TiKV (written), and final import completion (imported). Essential for diagnosing bottlenecks in bulk data loading workflows and identifying which stage is limiting overall import performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"restored\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"written\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"imported\"}[2m])) by(task_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 181, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Delete Range Task Status", "group": "GC", "used_for": "Used for monitoring the status and results of delete range tasks in TiDB's garbage collection process. Helps diagnose issues with range deletion operations, track task completion rates, and identify failures in cleaning up deleted data from TiKV.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*pipelined-dml.*\"}) by (type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 2534, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages in TiFlash-Proxy's gRPC communication. Helps diagnose network efficiency and identify whether Raft messages are being properly batched, which is critical for optimizing replication performance and reducing network overhead between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2533, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile and average batch sizes of gRPC requests and responses in TiFlash-Proxy. Essential for diagnosing gRPC communication efficiency and identifying potential batching issues that could impact TiFlash query performance or network utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Process Stat Cpu Usage", "group": "Storage", "used_for": "Used for monitoring CPU usage of TiKV storage processes across cluster instances. Essential for identifying CPU bottlenecks, resource contention, and determining if storage operations are CPU-bound or require scaling.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's Raft consensus protocol. Critical for diagnosing read performance issues related to Raft leader confirmation, especially when experiencing slow consistent reads or stale read detection delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Delivered KV Count", "group": "Import Into", "used_for": "Used for monitoring the total number of key-value pairs delivered during IMPORT INTO operations in TiDB. Helps track data ingestion progress and diagnose slow import performance by identifying bottlenecks in KV pair delivery rates.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_import_block_deliver_kv_pairs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 74, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index propose wait operations in TiKV's storage engine, tracking both p99 and p9999 percentiles. Critical for diagnosing delays in Raft read index proposals, which can indicate leader election issues, network problems, or Raft consensus bottlenecks affecting read performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Full compaction duration seconds", "group": "Storage", "used_for": "Used for monitoring the duration of full compaction operations in TiKV's storage engine at p99 and p9999 percentiles. Critical for diagnosing storage performance degradation and identifying when full compactions are taking abnormally long, which can impact write performance and storage space reclamation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 182, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Push Task Duration 95", "group": "GC", "used_for": "Used for monitoring the 95th percentile latency of pushing range tasks from TiDB to TiKV workers, particularly for garbage collection operations. Critical for diagnosing GC performance bottlenecks and identifying delays in task distribution that could impact cluster cleanup efficiency.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*pipelined-dml.*\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 325, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Coprocessor Response Size", "group": "Follower Read", "used_for": "Used for monitoring the size of data responses returned from TiKV coprocessor requests to TiDB, showing average, p50, p80, and p99 percentiles. Essential for diagnosing network bandwidth issues, identifying queries that transfer excessive data, and understanding the I/O characteristics of follower read operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_distsql_copr_resp_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.5, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 309, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Ongoing Internal Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of TiDB's internal transactions, such as those used for GC (garbage collection) and statistics updates. Critical for identifying when background maintenance operations are taking too long and potentially impacting cluster performance or causing resource contention.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"internal\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 319, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Data Read/Encode/Deliver Average Duration", "group": "Import Into", "used_for": "Used for monitoring the average latency of three key stages in TiDB's IMPORT INTO operation: reading source data, encoding rows, and delivering blocks to TiKV. Essential for diagnosing performance bottlenecks during bulk data import operations and identifying which stage (read, encode, or deliver) is causing slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "rate(tidb_import_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])", "rate(tidb_import_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]) / rate(tidb_import_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])", "rate(tidb_import_row_read_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_read_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 317, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Encoded/Delivered/Imported Data Size", "group": "Import Into", "used_for": "Used for monitoring the data volume processed during IMPORT INTO operations in TiDB, tracking encoded data size, delivered bytes to TiKV, and total imported data per task. Essential for diagnosing import performance bottlenecks, verifying data transfer completeness, and identifying tasks with abnormal data processing rates.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"restored\"}", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"written\"}", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"total_restore\"}) by(task_id)", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", state=\"imported\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's Raft consensus protocol, tracking both P99 and P9999 percentiles. Critical for diagnosing read performance issues related to Raft leader lease verification and identifying when follower reads experience delays in confirming the leader's commit index.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's async apply worker threads, which handle Raft log application to the storage engine. Essential for diagnosing write performance bottlenecks and identifying whether apply threads are saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 320, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Import/Normal Mode", "group": "Import Into", "used_for": "Used for monitoring whether TiKV is operating in import mode or normal mode by tracking RocksDB's hard pending compaction bytes limit configuration. Essential during data import operations to verify the cluster has switched to the appropriate mode for optimizing bulk data loading performance versus normal transactional workloads.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763571996, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of writing data to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sort tasks that involve cloud storage, particularly when investigating slow data import or DDL operations using global sort.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 64, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's scheduler worker threads across instances. Essential for identifying scheduler thread bottlenecks and diagnosing task scheduling performance issues in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 79, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Full compaction pause duration", "group": "Storage", "used_for": "Used for monitoring the pause duration during full compaction operations in TiKV's storage engine, tracking both p99 and p9999 latencies. Critical for diagnosing storage performance degradation and identifying when full compaction operations cause excessive pauses that may impact write availability and query latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 180, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Batch Client Unavailable Duration 95", "group": "Batch Client", "used_for": "Used for monitoring the 95th percentile duration when TiDB's batch client cannot send requests to TiKV due to unavailability. Critical for diagnosing connection issues, network problems, or TiKV overload situations that prevent TiDB from batching and sending requests efficiently.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 80, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Full compaction per-increment duration", "group": "Storage", "used_for": "Used for monitoring the duration of incremental full compaction operations in TiKV's storage engine at p99 and p9999 percentiles. Critical for diagnosing storage performance degradation and identifying when background compaction processes are taking unusually long, which can impact overall write performance and storage efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "__rate_interval", "additional_groupby", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 204, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Wait Connection Establish Duration", "group": "Batch Client", "used_for": "Used for monitoring the 99.99th percentile latency when TiDB batch clients wait to establish connections to TiKV. Critical for diagnosing connection pool exhaustion, network issues, or TiKV overload that causes delays in establishing new connections for batch operations.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 228, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Batch Receive Average Duration", "group": "Batch Client", "used_for": "Used for monitoring the average and P99 latency of batch receive operations between TiDB and TiKV clients. Critical for diagnosing network communication delays and identifying instances with slow batch RPC response times that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 203, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "No Available Connection Counter", "group": "Batch Client", "used_for": "Used for monitoring the rate of failed connection attempts when TiDB's batch client cannot obtain available connections to TiKV. Critical for diagnosing connection pool exhaustion issues and identifying when TiDB is unable to establish sufficient connections to TiKV nodes, which can lead to query failures or timeouts.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 83, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Local reader requests", "group": "Local Reader", "used_for": "Used for monitoring the rate of local read requests in TiKV, including successfully executed reads, stale reads, and rejected requests. Essential for diagnosing local reader performance issues and understanding why reads might be falling back to Raft consensus instead of being served locally.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4287, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's unified read pool threads, which handle data read operations. Essential for identifying read performance bottlenecks and determining if the read pool is CPU-saturated during heavy query workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 105, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC polling threads in TiFlash-Proxy instances. Essential for identifying excessive CPU consumption in gRPC communication layers, which may indicate network bottlenecks, high request volumes, or inefficient gRPC handling affecting TiFlash query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763571999, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Rate", "group": "Global Sort", "used_for": "**Used For:**\n\nUsed for monitoring the read throughput rate from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Essential for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, particularly when investigating slow data import or index building operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_global_sort_merge_sort_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\"}[2m]))/1024.0/1024.0" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 86, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's async apply worker threads, which handle Raft log application to the storage engine. Critical for diagnosing write performance bottlenecks and identifying whether apply threads are saturated during high write workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n [$__rate_interval]\n)) by (instance, name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash coprocessor threads across different priority levels (normal, high, and low). Essential for diagnosing TiFlash query processing bottlenecks and identifying whether coprocessor threads are saturated during analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1908, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash-Proxy's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether specific priority queues are saturated or experiencing CPU contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763571997, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the write throughput rate to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Essential for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage writes, particularly when investigating slow data import or index creation operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_global_sort_merge_sort_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\"}[2m]))/1024.0/1024.0" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 85, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's Raft store threads, which handle Raft consensus operations and region management. Critical for diagnosing performance bottlenecks related to Raft log processing, leader elections, and region split/merge operations that can impact cluster stability and write throughput.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiFlash-Proxy instances. Essential for identifying CPU bottlenecks in the storage engine layer and diagnosing performance issues related to data compaction, flushing, and I/O operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 239, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "99 Report Duration", "group": "TopSQL", "used_for": "Used for monitoring the 99th percentile latency of TopSQL report generation and submission in TiDB and TiKV. Critical for diagnosing performance issues in the TopSQL feature when reports are slow to generate or fail to be delivered to the monitoring system.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[5m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 243, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Total Report Count", "group": "TopSQL", "used_for": "Used for monitoring the total number of TopSQL reports generated by TiDB instances, broken down by report type and result status. Essential for verifying that TopSQL data collection is functioning correctly and diagnosing issues when query performance analysis data is missing or incomplete.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)", "sum(increase(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, result)", "sum(tikv_resource_metering_report_data_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"sent\"}) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763571998, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of reading data from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, particularly when investigating slow DDL operations or data import jobs using external storage.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 242, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Report Data Count Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the volume of TopSQL data being reported per minute from TiDB instances, broken down by data type. Helps diagnose TopSQL feature overhead and identify potential issues with query statistics collection or reporting pipeline congestion.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_report_data_total_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, job)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 241, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "CPU Profiling OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of CPU profiling operations in TiDB instances. Helps track the overhead and frequency of CPU profile collection, which is essential for understanding the performance impact of diagnostic tools like TopSQL when troubleshooting query performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_cpu_profile_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 305, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Ignore Event Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the rate of SQL queries ignored by TopSQL feature per minute, categorized by ignore type. Helps identify when query sampling is being skipped due to resource constraints or configuration limits, which may indicate TopSQL overhead issues or the need to adjust collection settings.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_ignored_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 87, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Raft async writer CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's Raft async writer threads, which handle asynchronous I/O operations for Raft log persistence. Critical for diagnosing write bottlenecks and identifying when Raft log processing becomes CPU-bound, especially during high write throughput scenarios.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2531, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's garbage collection worker threads. Essential for diagnosing GC performance issues and identifying instances where excessive GC activity may be impacting overall TiFlash performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 61, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's Raft store threads, which handle Raft consensus operations and data replication. Essential for diagnosing performance bottlenecks in TiFlash's data synchronization layer and identifying instances with high replication overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 40, "dashboard": "Dataplane Resource Overview", "title": "Pod Restart Count", "group": "Abnormal Pods", "used_for": "**Used For:**\n\nUsed for tracking the total number of restarts for each pod in the dataplane namespace. Helps identify unstable pods experiencing crashes, OOM kills, or liveness probe failures that may indicate resource constraints, configuration issues, or application bugs.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "cluster", "control_plane_info", "namespace" ], "promql": [ "sum(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum(kube_pod_container_status_restarts_total{control_plane_info=\"$control_plane_info\",cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 68, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read Index Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's ReadIndex worker threads, which handle read index requests from TiDB. Essential for diagnosing performance bottlenecks when TiFlash queries experience high latency or when investigating CPU saturation in the read path.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 91, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's background worker threads, which handle tasks like compaction, cleanup, and maintenance operations. Essential for identifying when background operations are consuming excessive CPU resources and potentially impacting foreground query performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 89, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Scheduler pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's scheduler thread pools, including both tokio async runtime threads and background worker threads. Essential for diagnosing transaction scheduling bottlenecks and identifying whether the scheduler is CPU-saturated during high concurrency workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_background.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 336, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Active Parallel Upload Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active parallel workers uploading sorted data during TiDB's global sort operations. Helps diagnose upload bottlenecks and resource utilization issues when global sort tasks experience high latency or slow data transfer to external storage.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_global_sort_upload_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 67, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy background worker threads across instances. Essential for identifying background task overhead and diagnosing performance issues caused by maintenance operations like compaction, cleanup, or data reorganization.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "Dataplane Resource Overview", "title": "Pod Abnormal Phase", "group": "Abnormal Pods", "used_for": "Used for identifying pods in abnormal states (excluding Running/Succeeded) within the dataplane namespace. Essential for quickly detecting pod failures, pending states, or crash loops that require immediate attention and troubleshooting.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\",namespace=\"$namespace\",phase!~\"Running|Succeeded\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 297, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiDB CPU Usage", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiDB worker instances, comparing actual CPU usage rate against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks in TiDB servers and determining if CPU resources need scaling during TTL (Time To Live) operations or heavy workloads.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"worker-tidb\"}" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 670, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region task worker pre-handle/generate snapshot CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's region task and region worker threads responsible for pre-handling requests and generating snapshots. Critical for diagnosing TiFlash region replication bottlenecks and identifying instances with excessive CPU consumption during snapshot generation or region task processing.", "component": "TIFLASH", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 36, "dashboard": "Dataplane Resource Overview", "title": "CPU Throttle", "group": "Abnormal Pods", "used_for": "Used for monitoring CPU throttling rates of pods in the dataplane, showing the percentage of time containers are throttled due to CPU limits. Critical for identifying pods experiencing performance degradation from insufficient CPU resources or overly restrictive CPU quotas.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 94, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Backup CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's backup worker threads. Essential for diagnosing performance issues during backup operations and identifying whether backup tasks are consuming excessive CPU resources that could impact cluster performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 92, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's garbage collection worker threads. Essential for diagnosing GC performance issues, identifying when GC operations are consuming excessive CPU resources, and determining if GC is impacting overall cluster performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL QPS By Type", "group": "TTL", "used_for": "Used for monitoring the queries per second (QPS) of TTL (Time To Live) operations in TiDB, broken down by SQL type and execution result. Helps identify TTL workload patterns, track the volume of expired data cleanup operations, and diagnose issues when TTL processing throughput is abnormal or failing.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 287, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Processed Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows processed by TiDB's Time-To-Live (TTL) feature, broken down by SQL operation type and result status. Essential for tracking TTL job performance, identifying processing bottlenecks, and ensuring expired data is being cleaned up at the expected rate.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 2, "dashboard": "Dataplane Resource Overview", "title": "Nodes", "group": "Node", "used_for": "Used for monitoring the distribution of Kubernetes nodes by instance type in the dataplane cluster. Helps operators understand cluster composition and identify node type imbalances or capacity planning needs.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_beta_kubernetes_io_instance_type)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 90, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's unified read pool threads, which handle both coprocessor requests and storage read operations. Essential for identifying read-heavy workloads and diagnosing performance bottlenecks when query or scan operations consume excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Dataplane Resource Overview", "title": "Node Load 1m", "group": "Node", "used_for": "Used for monitoring the 1-minute load average per CPU core across cluster nodes. Helps identify nodes experiencing high system load or resource contention, which can indicate performance bottlenecks or capacity issues requiring investigation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_load1_per_cpu:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 93, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Import CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's SST (Sorted String Table) import threads, which handle ingesting external SST files during data import operations like Lightning or BR restore. Essential for diagnosing performance bottlenecks and resource contention during bulk data import scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 303, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Delete Rows Per Hour", "group": "TTL", "used_for": "**Used For:**\n\nUsed for monitoring the number of expired rows successfully deleted by TiDB's TTL (Time To Live) feature per hour. Helps track TTL cleanup efficiency and identify potential issues with automatic data expiration and deletion operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 96, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "TSO worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's TSO (Timestamp Oracle) worker threads. Essential for diagnosing performance bottlenecks in timestamp allocation, which can impact transaction processing speed and overall cluster throughput when TSO workers become CPU-saturated.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's gRPC polling threads that handle client requests and inter-node communication. Essential for diagnosing whether gRPC threads are becoming a bottleneck during high concurrent workloads or network-intensive operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1070, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different request types from TiFlash-Proxy to PD (Placement Driver). Helps diagnose communication issues between TiFlash-Proxy and PD, particularly useful when investigating metadata operations or region information query delays.", "component": "PD", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763572005, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Stats Meta/Usage Updating Duration", "group": "Statistics & Plan Management", "used_for": "Used for monitoring the 99th percentile latency of statistics metadata and usage information updates in TiDB. Critical for diagnosing performance issues related to statistics collection and maintenance, which can impact query optimization and overall database performance when updates become slow.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_usage_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 71, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested CPU resources to allocatable CPU capacity on each Kubernetes node. Helps identify nodes approaching resource saturation and guides pod scheduling decisions to prevent over-commitment of CPU resources.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 302, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Insert Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows inserted per second by TiDB's Time-To-Live (TTL) feature. Helps track TTL background job activity and diagnose performance issues when TTL operations are inserting data back into tables or temporary storage.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 1215, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages sent from TiFlash-Proxy to PD (Placement Driver), categorized by message type. Essential for diagnosing communication issues between TiFlash-Proxy and PD, detecting heartbeat failures, and identifying potential cluster coordination problems that could affect TiFlash availability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 38, "dashboard": "Dataplane Resource Overview", "title": "Pod Last Terminated Reason", "group": "Abnormal Pods", "used_for": "**Used For:**\n\nUsed for identifying pods that have terminated abnormally in the specified cluster and namespace, showing the termination reason for each pod. Essential for quickly diagnosing container crashes, OOMKills, errors, or other non-normal termination events that may indicate application instability or resource constraints.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", reason!=\"Completed\"}) by (pod,reason)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 298, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Insert Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the hourly rate of rows inserted by TiDB's Time-To-Live (TTL) feature. Helps track TTL job activity and identify unusual insertion patterns or performance issues when TTL is automatically managing expired data cleanup and reinsertion.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1h]))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 30, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Usage", "group": "Node", "used_for": "Used for monitoring memory utilization ratio across individual nodes in the dataplane cluster. Helps identify nodes experiencing memory pressure or imbalanced memory consumption that could impact workload performance and stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_memory_utilisation:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1396, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiFlash-Proxy and PD (Placement Driver). Helps diagnose issues with region peer metadata synchronization and detect problems in the communication between TiFlash-Proxy and PD during cluster topology changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1069, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of PD (Placement Driver) requests made by TiFlash-Proxy instances. Helps identify communication patterns between TiFlash-Proxy and PD, and diagnose issues when metadata operations or region information queries are abnormally high or low.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 284, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Scan Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution (p50, p80, p90, p99) of TTL scan SELECT queries in TiDB. Essential for diagnosing performance issues with Time-To-Live background scanning operations and identifying when expired row cleanup is impacting query response times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))\n", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 32, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs in TiFlash-Proxy per server instance. Critical for diagnosing replication lag and write performance bottlenecks when TiFlash nodes struggle to keep up with data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 28, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Usage", "group": "Node", "used_for": "Used for monitoring CPU utilization across individual nodes in the dataplane cluster. Helps identify nodes experiencing high CPU load, enabling quick detection of resource bottlenecks and uneven workload distribution across the infrastructure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_cpu_utilisation:rate1m{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 97, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IME CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's In-Memory Engine (IME) threads. Helps identify CPU bottlenecks in the in-memory storage layer and diagnose performance issues related to memory-intensive operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested memory resources to allocatable memory capacity on each Kubernetes node. Helps identify nodes approaching memory resource limits and potential scheduling constraints before pods fail to deploy due to insufficient memory requests.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",unit=\"byte\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 98, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "IA CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's S3FIFO (Insertion-Aware) cache management threads. Helps identify if the adaptive cache eviction algorithm is consuming excessive CPU resources, which could indicate cache thrashing or inefficient memory management patterns.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 285, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TTL Delete Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution of TTL (Time To Live) delete queries in TiDB across multiple percentiles (p50, p80, p90, p99). Essential for diagnosing performance issues with automatic data expiration operations and identifying when TTL cleanup jobs are experiencing slow delete query execution.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 39, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of appending Raft logs to disk in TiFlash-Proxy's storage layer, showing p95, p99, and average durations. Critical for diagnosing write performance bottlenecks and disk I/O issues in TiFlash's Raft replication process, especially when data synchronization becomes slow.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 31, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of applying Raft logs in TiFlash-Proxy's Raft store, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication lag and identifying bottlenecks in the Raft log application process that could impact data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 23763573001, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Query Network Trasmission Bytes", "group": "Network Transmission", "used_for": "Used for monitoring the rate of network data transmission (in bytes per second) between TiDB and clients during query execution, broken down by transmission type. Essential for diagnosing network bottlenecks, identifying queries with large result sets, and troubleshooting slow data transfer issues between TiDB servers and applications.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_network_transmission{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1715, "dashboard": "[SharedScope]TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader", "group": "", "used_for": "Used for monitoring the number of Raft leader regions on each TiFlash-Proxy instance and detecting significant leader drops. Helps identify leader transfer issues, instance failures, or abnormal region migrations when leader counts decrease sharply (>10 regions in 2 minutes).", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 276, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "Scan Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) scan workers in TiDB. Helps identify performance bottlenecks in the TTL scanning process by showing which phase (e.g., scanning, deleting) consumes the most time, essential for optimizing expired data cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan_worker\"}[2m])) by (phase)" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 81, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "Concurrency manager max-ts", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring the concurrency manager's maximum timestamp (max-ts) in TiKV and its configured limit. Essential for diagnosing transaction conflicts and identifying when the concurrency manager approaches its capacity limit, which can impact transaction processing and cause performance degradation in high-concurrency scenarios.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "tikv_concurrency_manager_max_ts_limit", "tikv_concurrency_manager_max_ts" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 95, "dashboard": "[SharedScope]TiKV-Details-Cluster-Clinic", "title": "CDC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiCDC (Change Data Capture) worker threads running within TiKV instances. Essential for diagnosing CDC replication performance issues, identifying CPU bottlenecks in change data capture workloads, and determining if CDC operations are consuming excessive resources that may impact normal TiKV operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 245, "dashboard": "[SharedScope]TiDB-Worker-Cluster-Clinic", "title": "TiKV Stat Task OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of resource metering statistics tasks processed by TiKV instances. Helps diagnose TopSQL feature performance and identify issues with resource consumption tracking when analyzing query-level resource usage patterns.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_resource_metering_stat_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb-worker.json" }, { "panel_id": 23763572000, "dashboard": "[UserScope]TiDB-Cluster-Clinic", "title": "Ingest Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active ingest workers handling global sort operations in TiDB. Helps diagnose data import performance issues and worker resource allocation during large-scale data ingestion tasks like `IMPORT INTO` or `ADD INDEX`.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "keyspace_name", "tidb_cluster" ], "promql": [ "tidb_global_sort_ingest_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" } ]