[ { "panel_id": 43, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed PD client commands from TiDB instances, categorized by command type. Essential for identifying communication issues between TiDB and PD, helping diagnose cluster metadata access problems, TSO allocation failures, or network connectivity issues that could impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 116, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "95% Wait duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the 95th percentile wait time of coprocessor requests before execution across different TiKV stores. Essential for identifying stores experiencing queueing delays and diagnosing uneven load distribution or resource contention issues that cause requests to wait before processing.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 111, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the P99 wait duration of coprocessor requests in TiKV before they begin execution. Critical for diagnosing queueing delays and resource contention issues when coprocessor requests experience high latency due to thread pool saturation or scheduling bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2108, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the distribution of thread states (running, sleeping, waiting, etc.) across TiKV instances. Essential for diagnosing thread contention, resource saturation, and identifying instances with abnormal thread behavior that could indicate performance bottlenecks or deadlocks.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tikv_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 74, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Requests", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor requests per second in TiKV, broken down by request type. Essential for tracking coprocessor workload patterns and identifying which types of requests (table scan, index scan, etc.) are consuming cluster resources.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3195, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Requests", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) coprocessor requests processed by TiKV instances, broken down by vectorization type. Essential for tracking coprocessor workload intensity and identifying whether queries are using vectorized or non-vectorized execution, which helps diagnose query performance and resource utilization issues.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_dag_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (vec_type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2258, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput of TiKV threads, showing the top 20 threads by bytes read/written per second (filtered above 1KB/s). Essential for identifying which threads are performing heavy I/O operations and diagnosing storage bottlenecks or abnormal I/O patterns in TiKV instances.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, sum(rate(tikv_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 82, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB Get operation latency in TiKV, tracking maximum, P99, and P95 durations. Essential for diagnosing read performance issues and identifying slow point lookups in the storage engine that may impact query response times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 307, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests per second in TiDB, broken down by request type. Essential for tracking the usage and throughput of stale read operations, which allow reading slightly outdated data for improved performance and reduced latency in scenarios where strict consistency is not required.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_req_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 113, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the P99 latency of coprocessor request handling in TiKV, broken down by request type. Critical for diagnosing slow analytical queries and table scan performance issues, as coprocessor handles pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 306, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Hit/Miss OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of stale read requests that hit or miss the local replica in TiDB. Helps evaluate the effectiveness of stale read optimization and diagnose issues when stale reads frequently miss local replicas, requiring fallback to leader reads.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 53, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of KV client backoff operations in TiDB, categorized by backoff type. Essential for identifying retry patterns and diagnosing issues like region unavailability, lock conflicts, or TiKV pressure that force TiDB to back off and retry requests.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 117, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "95% Handle duration by store", "group": "Coprocessor Detail", "used_for": "Used for monitoring the 95th percentile latency of coprocessor request handling across individual TiKV stores. Essential for identifying performance bottlenecks and uneven load distribution when diagnosing slow query issues related to table scans, index lookups, or aggregation operations pushed down to TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by store", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of KV requests from TiDB to individual TiKV stores. Essential for identifying which specific TiKV stores are experiencing slow response times and diagnosing store-level performance bottlenecks or hotspots.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le, store))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 551, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Response Size", "group": "Coprocessor Overview", "used_for": "Used for monitoring the total rate of data returned by TiKV coprocessor requests across the cluster. Helps identify heavy scan operations, data transfer bottlenecks, and diagnose network bandwidth issues caused by large result sets from coprocessor queries.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 250, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache Error OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of region cache errors in TiDB's TiKV client, categorized by error type. Essential for diagnosing region routing issues, stale region metadata problems, and network connectivity failures that can cause query retries and performance degradation.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"err\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 57, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Num", "group": "Distsql", "used_for": "**Used For:**\n\nUsed for monitoring the number of keys scanned per distributed SQL query at different percentiles (p50, p90, max). Essential for identifying inefficient queries with excessive key scans, which can indicate missing indexes, full table scans, or poorly optimized query patterns causing performance degradation.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 77, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO Wait Duration", "group": "PD Client", "used_for": "**Used For:**\n\nUsed for monitoring the wait duration (p90, p99, p999) when TiDB requests timestamps from PD's TSO service. Critical for diagnosing transaction latency issues caused by PD client bottlenecks or network delays between TiDB and PD.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 126, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB write operation latency in TiKV, displaying maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or storage engine contention.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "TiDB-Cluster-Clinic", "title": "Lock Resolve OPS", "group": "KV Errors", "used_for": "Used for monitoring the rate of lock resolution operations in TiDB's transaction processing system. Essential for diagnosing transaction conflicts and identifying when the cluster is experiencing high contention, which can indicate write hotspots or long-running transactions blocking others.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 138, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation efficiency in TiKV by tracking where data is retrieved from (memtable, block cache, or L0 SST files). Essential for diagnosing read performance issues and understanding cache hit rates versus disk I/O patterns in the storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24763573286, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Quota", "group": "Coprocessor Overview", "used_for": "Used for monitoring the memory quota allocated to TiKV's coprocessor component across instances. Helps identify memory allocation patterns and potential memory pressure issues when coprocessor requests consume excessive resources, which can lead to OOM errors or query failures.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of PD client commands (excluding TSO operations) from TiDB instances at p90, p99, and p999 percentiles. Essential for diagnosing communication delays between TiDB and PD, helping identify network issues, PD overload, or metadata operation bottlenecks that impact query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"tso|tso_async_wait\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Backoff Duration", "group": "KV Errors", "used_for": "Used for monitoring the duration of KV client backoff operations at different percentiles (p80, p99, p999) when TiDB encounters errors communicating with TiKV. Essential for diagnosing performance degradation caused by region unavailability, lock conflicts, or other transient TiKV errors that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 60, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Partial QPS", "group": "Distsql", "used_for": "Used for monitoring the rate of partial distributed SQL scan operations per second in TiDB. Helps identify when queries are performing partial scans (scanning subset of data), which is useful for diagnosing query execution patterns and potential performance issues related to data distribution or query optimization.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_scan_keys_partial_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 263, "dashboard": "TiDB-Cluster-Clinic", "title": "RPC Layer Latency", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile network latency between TiDB and TiKV stores at the RPC layer. Critical for diagnosing network-related performance issues, identifying slow TiKV nodes, and distinguishing between storage engine delays versus network transmission delays in KV request processing.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store))", "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (le, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 7594, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory lock checking duration", "group": "Coprocessor Detail", "used_for": "Used for monitoring the time spent checking memory locks during coprocessor operations in TiKV. Essential for diagnosing performance bottlenecks related to memory lock contention when processing coprocessor requests, particularly during high-concurrency analytical queries.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_mem_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 308, "dashboard": "TiDB-Cluster-Clinic", "title": "Stale Read Req Traffic", "group": "KV Request", "used_for": "Used for monitoring the network traffic volume of stale read requests in TiDB, broken down by result status and direction (sent/received). Essential for tracking the I/O impact of stale read operations and diagnosing performance issues when using follower reads or historical data queries.", "component": "TIDB", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_stale_read_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, direction)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 52, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of key scanning operations performed by TiKV coprocessor requests. Helps diagnose coprocessor performance issues and identify queries causing excessive key scans, which can indicate inefficient query patterns or missing indexes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO RPC Duration", "group": "PD Client", "used_for": "Used for monitoring the tail latency of TSO (Timestamp Oracle) RPC requests from TiDB to PD at p99, p999, and p9999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication issues that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99, 999, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 16, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the 99th percentile and maximum latency of coprocessor requests in TiKV, broken down by request type. Critical for diagnosing slow analytical queries and table scan performance issues, as coprocessor handles pushed-down computations from TiDB.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "TiDB-Cluster-Clinic", "title": "Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of partial results generated during distributed SQL query execution in TiDB. Helps diagnose query performance issues related to data distribution and parallel processing, where higher partial counts may indicate inefficient query plans or data skew across TiKV regions.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 129, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation rates in TiKV, tracking total seeks, successful seeks, and next operations. Essential for diagnosing read performance issues and identifying inefficient scan patterns that may indicate suboptimal query execution or excessive iterator usage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Duration 99 by type", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of different KV request types (Get, Scan, Prewrite, Commit, etc.) sent from TiDB to TiKV. Essential for identifying which specific KV operation types are experiencing high latency and diagnosing TiDB-TiKV communication performance bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", store!=\"0\", scope=\"false\"}[2m])) by (le,type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3264, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total DAG Executors", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of DAG (Directed Acyclic Graph) executor operations in TiKV's coprocessor, broken down by executor type. Helps identify which coprocessor operations (like TableScan, Selection, Aggregation) are most frequently used and diagnose performance bottlenecks in pushdown query execution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 175, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Cache", "group": "Distsql", "used_for": "Used for monitoring the coprocessor cache hit/miss rates in TiDB's distributed SQL execution. Helps identify whether coprocessor requests are being efficiently cached, which is crucial for optimizing query performance and reducing unnecessary data transfers from TiKV.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 172, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request OPS", "group": "KV Request", "used_for": "Used for monitoring the operations per second (OPS) of different KV request types sent from TiDB to TiKV, broken down by TiDB instance. Essential for identifying which TiDB nodes are generating high request volumes and diagnosing uneven load distribution or request type anomalies that may indicate application issues or hotspots.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"false\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 220, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS by Type", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded by TiDB to other TiKV nodes, broken down by request type and result status. Essential for diagnosing follower read performance, identifying forwarding bottlenecks, and understanding request routing patterns when TiDB needs to redirect requests to appropriate TiKV replicas.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 164, "dashboard": "TiDB-Cluster-Clinic", "title": "Region Cache OK OPS", "group": "KV Request", "used_for": "Used for monitoring the successful operations per second on TiDB's region cache, broken down by operation type. Helps assess region cache efficiency and identify when cache operations are performing normally versus experiencing failures that could impact query routing performance.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 139, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiKV's RocksDB storage engine, including successful writes (self/other), timeouts, and WAL writes. Essential for diagnosing write throughput issues, identifying write bottlenecks, and detecting write timeout problems that may indicate storage layer performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "TiDB-Cluster-Clinic", "title": "TiClient Region Error OPS", "group": "KV Errors", "used_for": "**Used For:**\n\nUsed for monitoring the rate of region-related errors encountered by TiDB when accessing TiKV, broken down by error type (including server_is_busy). Critical for diagnosing region availability issues, hotspot problems, and TiKV overload conditions that impact query execution.", "component": "TIDB", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2118, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total RocksDB Perf Statistics", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of internal delete operations skipped by RocksDB during coprocessor requests in TiKV. Helps diagnose storage engine efficiency and identify potential issues with MVCC garbage collection or compaction performance that may impact coprocessor query execution.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2661, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for identifying TiKV threads experiencing excessive involuntary context switches (>100/sec), which indicates CPU contention or resource starvation. Critical for diagnosing performance degradation caused by thread scheduling issues, where threads are forcibly preempted by the OS rather than yielding voluntarily.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, max(rate(tikv_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 100)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 223, "dashboard": "TiDB-Cluster-Clinic", "title": "Replica Selector Failure Per Second", "group": "KV Errors", "used_for": "Used for monitoring the rate of replica selector failures per second in TiDB's TiKV client, categorized by failure type. Critical for diagnosing issues with TiDB's ability to select appropriate TiKV replicas for read/write operations, which can indicate network problems, region unavailability, or load balancing issues.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_replica_selector_failure_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3263, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations in TiKV's coprocessor, broken down by operation type (tag). Essential for diagnosing index scan performance issues and understanding which types of index operations (e.g., processed keys, total keys) are contributing to coprocessor workload.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 122, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations across different column families (CF) in TiKV's Coprocessor. Helps diagnose query performance issues by identifying which column families are experiencing high scan throughput and detecting potential full table scan problems.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag,cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details (Table Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of table scan operations in TiKV's coprocessor, broken down by scan type (e.g., full table scan, index scan). Essential for identifying inefficient queries that perform excessive table scans and diagnosing coprocessor performance bottlenecks caused by scan-heavy workloads.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "TiDB-Cluster-Clinic", "title": "PD TSO OPS", "group": "PD Client", "used_for": "Used for monitoring the throughput of TSO (Timestamp Oracle) requests from TiDB to PD, measured in operations per second. Essential for identifying TSO request volume patterns and detecting potential bottlenecks in timestamp allocation, which is critical for transaction processing performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))", "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 20, "dashboard": "TiDB-Cluster-Clinic", "title": "PD Client CMD OPS", "group": "PD Client", "used_for": "Used for monitoring the operations per second (OPS) of different PD client commands from TiDB instances, excluding TSO requests. Helps identify which PD API calls (like GetRegion, GetStore) are most frequently used and diagnose potential bottlenecks in TiDB's communication with PD.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"tso\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 41, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Seconds 999", "group": "Distsql", "used_for": "Used for monitoring the 99.9th percentile latency of coprocessor requests sent from TiDB to TiKV. Critical for identifying tail latency issues in distributed query execution, particularly when diagnosing slow analytical queries or table scans that push computation down to TiKV.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_request_seconds_bucket{store!=\"0\", scope=\"false\", type=\"Cop\", k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 554, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Ops Details by CF (Index Scan)", "group": "Coprocessor Detail", "used_for": "Used for monitoring the rate of index scan operations across different column families (CF) in TiKV's coprocessor. Helps diagnose index scan throughput patterns and identify which column families are handling the most index read operations, useful for troubleshooting coprocessor performance issues related to index queries.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[2m])) by (tag,cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Keys Partial Num", "group": "Distsql", "used_for": "Used for monitoring the number of keys scanned in partial results during distributed SQL execution at p80 and p90 percentiles. Helps diagnose query efficiency issues by identifying queries that scan excessive keys, which may indicate missing indexes or inefficient query patterns causing performance degradation.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 219, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Request Forwarding OPS", "group": "KV Request", "used_for": "Used for monitoring the rate of KV requests forwarded between TiKV stores in TiDB. Helps identify request routing patterns and diagnose issues when requests need to be redirected from one store to another, which may indicate leader transfers, region migrations, or store failures.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_forward_request_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (from_store, to_store, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 262, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Region Duration", "group": "KV Request", "used_for": "Used for monitoring the 99th percentile latency of loading region information into TiDB's region cache from PD. Critical for diagnosing slow KV request issues caused by region metadata lookup delays, especially when clients need to locate which TiKV node stores specific data ranges.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3128, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Request Errors", "group": "Coprocessor Overview", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV, broken down by error reason. Essential for diagnosing query execution failures and identifying specific error patterns (such as lock conflicts, region errors, or timeout issues) that may be impacting application performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 125, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiKV, tracking maximum, P99, and P95 durations. Essential for diagnosing read performance issues caused by inefficient data scanning or index lookups in the storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2660, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring threads with high voluntary context switch rates in TiKV, indicating threads frequently yielding CPU due to I/O waits or blocking operations. Helps identify I/O bottlenecks and threads experiencing excessive waiting, which can impact overall TiKV performance and throughput.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, max(rate(tikv_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) > 200)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3129, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "KV Cursor Operations", "group": "Coprocessor Overview", "used_for": "Used for monitoring the number of keys scanned per coprocessor request at the 99th percentile and maximum levels in TiKV. Essential for diagnosing inefficient query patterns, identifying full table scans, and detecting coprocessor operations that scan excessive keys causing performance degradation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) ", "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, req)) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 137, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations per second in TiKV's RocksDB engine. Critical for diagnosing write durability overhead and identifying potential I/O bottlenecks when WAL sync frequency becomes excessive or irregular.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 130, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of writing to RocksDB's Write-Ahead Log (WAL) in TiKV. Critical for diagnosing disk I/O performance issues and identifying storage bottlenecks that impact write operations, as high WAL write duration directly affects transaction commit latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 159, "dashboard": "TiDB-Cluster-Clinic", "title": "Async TSO Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of asynchronous TSO (Timestamp Oracle) requests from TiDB to PD at p90, p99, and p999 percentiles. Critical for diagnosing transaction start delays and identifying PD client communication bottlenecks that impact overall transaction performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso_async_wait\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 222, "dashboard": "TiDB-Cluster-Clinic", "title": "Request forwarded status", "group": "PD Client", "used_for": "Used for monitoring whether TiDB's PD client requests are being forwarded to other PD nodes instead of being handled directly by the connected PD leader. Helps diagnose PD leadership changes, network routing issues, or PD cluster instability that causes request redirection overhead.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "pd_client_request_forwarded_status" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2453, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction guard actions", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of compaction guard actions in TiKV's RocksDB storage engine across default and write column families. Helps diagnose write stall issues and compaction throttling behavior when TiKV is under heavy write pressure or experiencing disk I/O bottlenecks.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_compaction_guard_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=~\"default|write\"}[2m])) by (cf, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 248, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Fail OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of failed HTTP requests from TiDB to PD (Placement Driver). Essential for identifying communication issues between TiDB and PD, helping diagnose cluster coordination problems, metadata access failures, and potential network or PD service availability issues.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result!~\"200.*\"}[2m])) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 246, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request Duration", "group": "PD Client", "used_for": "Used for monitoring the latency of HTTP requests from TiDB to PD (Placement Driver) at different percentiles (p90, p99, p999). Essential for diagnosing communication issues between TiDB and PD, identifying network bottlenecks, or detecting PD performance degradation that impacts cluster metadata operations.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 247, "dashboard": "TiDB-Cluster-Clinic", "title": "PD HTTP Request OPS", "group": "PD Client", "used_for": "Used for monitoring the rate of HTTP API requests from TiDB servers to PD (Placement Driver). Helps identify communication patterns between TiDB and PD, diagnose potential PD client bottlenecks, and detect abnormal request spikes that may indicate metadata access issues or cluster coordination problems.", "component": "TIDB", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_pd_api_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 136, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiKV, showing maximum, P99, and P95 durations. Critical for diagnosing write amplification issues, storage performance degradation, and identifying when background compaction operations are impacting foreground query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 135, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the latency of Write-Ahead Log (WAL) file synchronization operations in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues, as high WAL sync latency directly impacts transaction commit times and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile duration of schema loading operations in TiDB. Critical for diagnosing slow DDL execution, schema synchronization delays, and identifying instances experiencing performance issues when fetching or updating table metadata from TiKV.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 128, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiKV, broken down by operation type (flush, compaction levels). Essential for diagnosing storage engine performance issues, identifying excessive compaction activity that may impact write throughput, and understanding background I/O patterns in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 140, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of reading SST (Sorted String Table) files from disk in TiKV's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing storage I/O performance issues and identifying slow disk reads that impact query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 102, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the block cache memory usage across different column families in TiKV's RocksDB storage engine. Essential for diagnosing memory pressure issues, cache efficiency problems, and optimizing RocksDB block cache configuration to balance memory usage and read performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 315, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB Wait TSO Future Duration", "group": "PD Client", "used_for": "Used for monitoring the duration TiDB waits when a TSO (timestamp oracle) timestamp is in the future, tracking p90, p99, and p999 latencies. Critical for diagnosing clock synchronization issues between TiDB and PD servers, which can cause transaction delays and performance degradation.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tidb_tikvclient_ts_future_wait_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le) / sum(rate(tidb_tikvclient_ts_future_wait_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 251, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema Action Duration", "group": "Schema Load", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency of schema loading operations in TiDB, broken down by action type (e.g., reload, diff). Critical for diagnosing DDL performance issues and identifying slow schema synchronization that can impact query execution and cluster stability.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, action))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 28, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Schema OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema loading operations in TiDB instances, categorized by operation type. Essential for diagnosing schema synchronization issues, identifying excessive schema reloads, and troubleshooting DDL-related performance problems across the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_load_schema_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2451, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency and reasons why RocksDB triggers compaction operations in TiKV. Helps diagnose storage engine behavior and identify whether compactions are triggered by normal operations (like level size limits) or abnormal conditions (like too many files), which is critical for understanding write amplification and I/O performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 467, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write throughput and data insertion rate of RocksDB's block cache in TiKV. Essential for analyzing cache utilization patterns and diagnosing memory pressure issues when investigating storage layer performance problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 314, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Cache OPS", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema cache operations in TiDB, broken down by action and type. Helps diagnose schema loading performance issues, cache hit/miss patterns, and identify excessive schema reloads that could impact query performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_domain_infocache_counters{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (action,type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 29, "dashboard": "TiDB-Cluster-Clinic", "title": "Schema Lease Error OPM", "group": "Schema Load", "used_for": "Used for monitoring the rate of schema lease errors per minute in TiDB instances. Critical for detecting schema synchronization issues between TiDB servers, which can cause query failures or inconsistent metadata when the schema lease mechanism fails to maintain consistency across the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_schema_lease_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 157, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Data From Cached Table Duration", "group": "Schema Load", "used_for": "Used for monitoring the 99th percentile latency when loading data from TiDB's cached tables feature. Critical for diagnosing performance issues with table cache operations and identifying instances where cache loading becomes a bottleneck for query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572002, "dashboard": "TiDB-Cluster-Clinic", "title": "Lease Duration", "group": "Schema Load", "used_for": "Used for monitoring the remaining time until TiDB's schema lease expires. Critical for diagnosing schema synchronization issues and detecting potential schema lease expiration problems that could cause DDL operations to fail or stall.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_domain_lease_expire_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} - time()" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 86, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the write throughput in TiKV's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written per second. Essential for diagnosing write performance bottlenecks, identifying I/O saturation issues, and understanding storage layer write patterns during high-load scenarios.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Backfill Data Rate", "group": "DDL", "used_for": "Used for monitoring the rate at which DDL operations backfill data during index creation in TiDB. Essential for tracking DDL job progress and diagnosing slow index building performance, helping identify bottlenecks when adding indexes to large tables.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_add_index_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 80, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the hit rate of RocksDB's block cache in TiKV, including overall cache hits, data block hits, and filter block hits. Essential for evaluating cache effectiveness and diagnosing performance issues caused by insufficient cache memory or poor cache utilization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 131, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys stored in each RocksDB column family within TiKV. Helps assess data distribution across column families and identify potential storage imbalances or unexpected data growth patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 85, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of RocksDB storage engine in TiKV, tracking both direct bytes read and iterator-based reads. Essential for diagnosing read I/O bottlenecks, understanding read workload patterns, and identifying potential performance issues in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Worker Duration 99", "group": "DDL", "used_for": "Used for monitoring the 99th percentile latency of DDL worker operations in TiDB, broken down by operation type and action. Critical for diagnosing slow schema changes, identifying DDL bottlenecks, and understanding why table alterations or schema modifications are taking longer than expected.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type, action, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 132, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiKV's RocksDB storage engine, including keys read, keys written, and corrupt keys detected during compaction. Essential for diagnosing data access patterns, write throughput issues, and identifying potential data corruption problems in the storage layer.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the hit rate of RocksDB memtable lookups in TiKV. Helps diagnose read performance issues by showing how effectively recent writes are being served from memory before flushing to disk, with low hit rates potentially indicating memory pressure or suboptimal memtable configuration.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Duration 95", "group": "DDL", "used_for": "Used for monitoring the 95th percentile latency of DDL (Data Definition Language) operations in TiDB, including schema changes and index creation. Critical for diagnosing slow DDL performance issues and understanding the impact of schema modifications on cluster operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 62, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Waiting Jobs Count", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs waiting in the queue across TiDB instances. Critical for identifying DDL operation bottlenecks, schema change delays, and potential blocking issues that could impact database schema modifications.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_waiting_jobs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 66, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Self Version Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL owner updating its version information in the cluster. Critical for diagnosing DDL coordination issues and detecting delays in schema change propagation across TiDB instances.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 90, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction I/O throughput in TiKV, tracking bytes read during compaction, bytes written after compaction, and flush write operations. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification that may impact overall cluster I/O efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 264, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL job table duration", "group": "DDL", "used_for": "Used for monitoring the 95th percentile duration of DDL job table operations in TiDB. Critical for diagnosing slow DDL execution and identifying performance bottlenecks in schema changes, helping pinpoint whether delays occur during job table reads/writes or metadata operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", }[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 103, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables across different column families in TiKV. Essential for diagnosing memory pressure issues and understanding write buffer usage, as excessive memtable size can indicate write stalls or memory configuration problems.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables-all\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571993, "dashboard": "TiDB-Cluster-Clinic", "title": "Distributed Task Running Subtask Distribution on TiDB Nodes", "group": "Dist Execute Framework", "used_for": "Used for monitoring the distribution of active subtasks across TiDB nodes in the distributed execution framework. Helps identify workload imbalance and diagnose issues when certain nodes are overloaded or underutilized during distributed task execution.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"pending|running|reverting|revert_pending|paused\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 468, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiKV's RocksDB storage engine, including overall blocks, data blocks, and filter blocks. Essential for understanding cache population patterns and diagnosing issues related to cache churn or excessive memory allocation in the block cache.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 517, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiKV's RocksDB storage engine. Critical for identifying long-running transactions or stale snapshots that can block compaction, cause disk space bloat, and degrade overall storage performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data read per RocksDB read operation in TiKV, showing maximum, 99th, and 95th percentile values. Helps diagnose read amplification issues and identify whether queries are reading excessive amounts of data, which can indicate inefficient data access patterns or compaction problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 265, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL Running Job Cnt", "group": "DDL", "used_for": "Used for monitoring the number of DDL (Data Definition Language) jobs currently running in TiDB. Helps identify DDL operation bottlenecks, track schema change progress, and diagnose issues when DDL operations are stuck or accumulating.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_running_job_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 518, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiKV's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O and degrade read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 192, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL backfill progress in percentage", "group": "DDL", "used_for": "Used for tracking the completion percentage of DDL backfill operations in TiDB, including index creation and column modifications. Essential for monitoring long-running schema changes and estimating when DDL operations will complete.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tidb_ddl_backfill_percentage_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"modify_column\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 134, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiKV's RocksDB storage engine, showing maximum, P99, and P95 values. Helps diagnose write amplification issues and identify whether write operations are batching efficiently or writing unexpectedly large amounts of data per operation.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 863, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression efficiency of data stored in TiKV's RocksDB engine across different column families and LSM tree levels. Essential for evaluating storage space utilization and identifying compression issues that could lead to excessive disk usage or degraded I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572001, "dashboard": "TiDB-Cluster-Clinic", "title": "Add Index Scan Rate", "group": "DDL", "used_for": "Used for monitoring the row scanning throughput (rows per second) during ADD INDEX operations in TiDB at different percentiles (p50, p95, p999). Essential for evaluating DDL performance and diagnosing slow index creation issues by identifying whether the scanning phase is the bottleneck.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_ddl_scan_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2002, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in RocksDB's LSM-tree structure for each column family. Essential for diagnosing compaction issues, identifying level imbalances, and understanding storage engine health that can impact read/write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL META OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) metadata operations per minute in TiDB, tracking the rate of schema changes like CREATE, ALTER, and DROP operations. Essential for diagnosing DDL performance issues, identifying schema change bottlenecks, and understanding the frequency of metadata modifications across TiDB instances.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_ddl_worker_operation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 322, "dashboard": "TiDB-Cluster-Clinic", "title": "Task Status", "group": "Dist Execute Framework", "used_for": "Used for monitoring the status distribution of distributed tasks in TiDB's distributed execution framework. Helps identify task execution states (pending, running, completed, failed) and diagnose issues with distributed operations like DDL, import, or analytics workloads.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_disttask_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2452, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for identifying the root causes of write stalls in TiKV's RocksDB storage engine by tracking different stall reason types. Essential for diagnosing write performance degradation and understanding whether stalls are caused by memtable limits, L0 file accumulation, pending compaction bytes, or other RocksDB backpressure mechanisms.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 326, "dashboard": "TiDB-Cluster-Clinic", "title": "Dist Task Subtasks", "group": "Dist Execute Framework", "used_for": "Used for monitoring the status and count of distributed task subtasks in TiDB's distributed execution framework. Helps track task completion rates and identify failed or stuck subtasks by comparing completed subtasks (succeed/failed/canceled/reverted) against total subtasks across different task types.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=~\"succeed|failed|canceled|reverted|revert_failed\"}) by (task_id, task_type)", "sum(tidb_disttask_subtask_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (task_id, task_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2003, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST (Sorted String Table) file ingestion operations in TiKV's RocksDB engine. Critical for diagnosing performance issues during snapshot recovery, data import operations, or when TiKV ingests external SST files, as high latency indicates I/O bottlenecks or storage layer problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 516, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiKV instances. Helps identify potential memory leaks or snapshot accumulation issues that can impact performance and resource usage when snapshots are not properly released.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2381, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for tracking the frequency of RocksDB stall condition changes in TiKV's storage engine. Essential for diagnosing write throttling and flow control issues that occur when RocksDB enters stall states due to memtable limits, L0 file accumulation, or pending compaction bytes exceeding thresholds.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 127, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the total bytes of data waiting to be compacted in RocksDB column families. Critical for identifying compaction backlogs that can lead to read amplification, write stalls, and degraded I/O performance in TiKV.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 65, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Handle Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the maximum latency of DDL owner operations when handling schema synchronization across TiDB nodes. Critical for diagnosing DDL execution delays and identifying bottlenecks in schema change propagation, especially when DDL operations appear stuck or take longer than expected.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12712, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ingestion picked level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of LSM tree levels where ingested SST files are placed in TiKV's RocksDB engine. Helps diagnose ingestion efficiency and potential compaction bottlenecks by showing whether files are being placed at optimal levels during bulk data imports or snapshot operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_engine_ingestion_picked_level_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 190, "dashboard": "TiDB-Cluster-Clinic", "title": "DDL OPM", "group": "DDL", "used_for": "Used for monitoring DDL (Data Definition Language) operations per minute in TiDB, broken down by operation type (e.g., CREATE, ALTER, DROP). Essential for tracking schema change frequency and identifying periods of heavy DDL activity that may impact cluster performance or cause lock contention.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_ddl_handle_job_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 87, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiKV's RocksDB storage engine, tracking maximum, P99, and P95 latencies when writes are temporarily blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when RocksDB flow control mechanisms are throttling write operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 64, "dashboard": "TiDB-Cluster-Clinic", "title": "Deploy Syncer Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL syncer deployment operations in TiDB. Helps diagnose performance issues during schema synchronization setup, particularly when DDL changes are being distributed across the cluster.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 46, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Duration", "group": "Statistics", "used_for": "Used for monitoring the duration of automatic statistics collection (auto-analyze) operations in TiDB at the 80th and 95th percentiles. Essential for diagnosing performance issues related to automatic table statistics updates, which can impact query optimization and overall cluster performance during background maintenance tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12896, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow", "group": "Raft Engine", "used_for": "Used for monitoring the data flow rates in TiKV's Raft Engine, including write throughput and background rewrite operations by type. Essential for understanding Raft log write performance and identifying potential I/O bottlenecks or excessive rewrite activity that could impact cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_background_rewrite_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12892, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Operation", "group": "Raft Engine", "used_for": "Used for monitoring the operation rates of Raft Engine I/O activities, including write apply, entry reads, and message reads. Helps identify I/O throughput patterns and diagnose performance bottlenecks in TiKV's Raft log storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_apply_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_read_entry_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(raft_engine_read_message_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Inaccuracy Rate", "group": "Statistics", "used_for": "**Used For:**\n\nUsed for monitoring the inaccuracy rate of table statistics at different percentiles (p50, p90, p99) in TiDB. Critical for identifying when statistics become stale or inaccurate, which can lead to suboptimal query execution plans and performance degradation requiring statistics updates.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [ 50, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 47, "dashboard": "TiDB-Cluster-Clinic", "title": "Auto Analyze Queries Per Minute", "group": "Statistics", "used_for": "Used for monitoring the rate of automatic statistics collection operations in TiDB per minute, broken down by analysis type. Helps identify when auto-analyze jobs are running and diagnose potential performance impacts from excessive statistics gathering or detect when auto-analyze is not functioning as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_auto_analyze_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 233, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Healthy Distribution", "group": "Statistics", "used_for": "Used for monitoring the health status distribution of table statistics in TiDB. Essential for identifying stale or outdated statistics that can lead to suboptimal query execution plans and performance degradation.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_statistics_stats_healthy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 93, "dashboard": "TiDB-Cluster-Clinic", "title": "Update Stats OPS", "group": "Statistics", "used_for": "Used for monitoring the rate of statistics update operations in TiDB, broken down by update type. Essential for diagnosing statistics collection performance issues and understanding when auto-analyze jobs are running, which can impact query optimization and cluster performance.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_update_stats_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 71, "dashboard": "TiDB-Cluster-Clinic", "title": "Pseudo Estimation OPS", "group": "Statistics", "used_for": "Used for monitoring the rate of queries using pseudo statistics (statistics generated without actual table analysis) in TiDB. High values indicate tables lacking proper statistics, which can lead to suboptimal query plans and performance degradation requiring ANALYZE TABLE operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_pseudo_estimation_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12893, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration", "group": "Raft Engine", "used_for": "Used for monitoring the write latency of TiKV's Raft Engine, which persists Raft logs to disk. Critical for diagnosing consensus replication performance issues and identifying disk I/O bottlenecks that could impact transaction commit times, with p95, p99, and p999 percentiles helping detect tail latency problems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.95, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12895, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Duration Breakdown (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency breakdown of Raft Engine write operations across preprocessing, leader writing, and apply stages in TiKV. Essential for diagnosing write performance bottlenecks and identifying which specific phase of the Raft write pipeline is causing slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12898, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Bytes / Written", "group": "Raft Engine", "used_for": "Used for monitoring the average, P99, and P999 bytes written per operation to TiKV's Raft Engine. Essential for diagnosing write amplification issues and understanding the size distribution of Raft log writes, which impacts storage efficiency and I/O performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(raft_engine_write_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(raft_engine_write_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12899, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "File Count", "group": "Raft Engine", "used_for": "Used for monitoring the number of log files, swap files, and recycled files in TiKV's Raft Engine storage layer. Essential for tracking file management health, detecting file accumulation issues, and diagnosing potential disk space or file cleanup problems that could impact Raft log persistence performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(raft_engine_log_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)", "avg(raft_engine_swap_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "avg(raft_engine_recycled_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12933, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "WAL Duration Breakdown (999%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99.9th percentile latency breakdown of Raft Engine's Write-Ahead Log (WAL) operations, including leader write, log sync, and log allocation phases. Critical for diagnosing Raft log write performance bottlenecks and identifying which specific WAL operation stage is causing tail latency issues in TiKV's consensus replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(raft_engine_write_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_sync_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_allocate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_rotate_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 235, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache OPS", "group": "Statistics", "used_for": "Used for monitoring the operations per second on TiDB's statistics cache, broken down by operation type (hits, misses, evictions). Essential for diagnosing statistics cache efficiency and identifying when cache misses may be causing query plan degradation or performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_stats_cache_op{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 229, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load QPS", "group": "Statistics", "used_for": "Used for monitoring the throughput of synchronous statistics loading operations in TiDB, including both successful loads and timeouts. Essential for diagnosing statistics collection performance issues and identifying when statistics loading is experiencing delays or failures that could impact query optimization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_statistics_sync_load_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_statistics_sync_load_timeout_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3523, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Live blob size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the total size of live (non-garbage) blob files in TiKV's Titan storage engine. Essential for tracking Titan's storage efficiency and identifying when blob garbage collection is needed to reclaim space from deleted or updated large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_live_blob_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12897, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Other Durations (99%)", "group": "Raft Engine", "used_for": "Used for monitoring the 99th percentile latency of miscellaneous Raft Engine operations in TiKV, including reading entries, reading messages, and purging old data. Essential for diagnosing performance bottlenecks in Raft log management operations that may impact consensus protocol efficiency and overall cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(raft_engine_read_entry_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_read_message_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_purge_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 234, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Cache Cost", "group": "Statistics", "used_for": "**Used For:**\n\nUsed for monitoring TiDB's statistics cache memory usage, tracking both current tracked items and total capacity. Essential for diagnosing memory pressure issues related to table statistics caching and determining if cache size adjustments are needed to optimize query plan generation.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"track\"}", "tidb_statistics_stats_cache_val{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"capacity\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the number of live and obsolete blob files in TiKV's Titan storage engine. Essential for tracking Titan's blob file accumulation and garbage collection effectiveness, helping diagnose storage space issues and GC performance problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_num_obsolete_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Entry Count", "group": "Raft Engine", "used_for": "Used for monitoring the average number of log entries stored in TiKV's Raft Engine, categorized by entry type. Helps diagnose Raft log accumulation issues, identify potential log compaction problems, and understand the distribution of different entry types that could impact storage and recovery performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(raft_engine_log_entry_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 162, "dashboard": "TiDB-Cluster-Clinic", "title": "New ETCD Session Duration 95", "group": "Owner", "used_for": "Used for monitoring the 95th percentile latency of establishing new ETCD sessions in TiDB's owner election mechanism. Critical for diagnosing coordination issues and detecting ETCD connectivity problems that could affect DDL operations and cluster metadata management.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, result))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572005, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Meta Updating Duration", "group": "Statistics", "used_for": "Used for monitoring the 99th percentile duration of statistics metadata updates in TiDB. Critical for diagnosing performance issues related to table statistics maintenance, which can impact query optimization and overall database performance when updates become slow.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_update_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3746, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob retrieval operations in TiKV's Titan storage engine. Essential for tracking Titan's blob file access patterns and diagnosing performance issues related to large value reads when Titan is enabled.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_get\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4655, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the blob cache memory usage in TiKV's Titan storage engine, showing the top 20 instances by column family. Essential for tracking Titan's blob file cache consumption and diagnosing memory pressure issues when large values are stored separately from the LSM tree.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "topk(20, avg(tikv_engine_blob_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by(cf, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 236, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Replayer Task OPM", "group": "Statistics", "used_for": "Used for monitoring Plan Replayer task execution rates in TiDB, tracking both dump and capture operations per minute along with registered tasks. Essential for diagnosing issues with SQL plan reproduction and troubleshooting query optimization problems, as Plan Replayer helps capture execution context for problematic queries.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)", "sum(rate(tidb_plan_replayer_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"capture\"}[2m])) by (result)", "avg(tidb_plan_replayer_register_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 230, "dashboard": "TiDB-Cluster-Clinic", "title": "Sync Load Latency P9999", "group": "Statistics", "used_for": "Used for monitoring the 99.99th percentile latency of statistics synchronous loading operations in TiDB, including both overall sync load time and stats reading time. Critical for diagnosing extreme tail latency issues in statistics collection that could impact query optimization and execution planning performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4020, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob cache hit", "group": "Titan - $titan_db", "used_for": "Used for monitoring the cache hit rate of Titan's blob cache in TiKV. Helps diagnose blob storage performance issues by showing how effectively the cache is serving blob reads, where low hit rates indicate potential memory pressure or inefficient caching that could impact large value query performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) / (sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_hit\"}[2m])) + sum(rate(tikv_engine_blob_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$titan_db\", type=\"blob_cache_miss\"}[2m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572006, "dashboard": "TiDB-Cluster-Clinic", "title": "Stats Loading Duration", "group": "Statistics", "used_for": "Used for monitoring the 99th percentile latency of statistics delta loading operations in TiDB. Critical for diagnosing performance issues related to statistics updates, which can impact query optimization and execution plan generation when loading takes too long.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_delta_load_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3557, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the storage space occupied by Titan blob files in TiKV, showing both live (active) and obsolete (garbage) blob file sizes. Essential for tracking Titan storage efficiency and identifying when garbage collection is needed to reclaim space from deleted large values.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_live_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_obsolete_blob_file_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 52, "dashboard": "TiDB-Cluster-Clinic", "title": "Meta Operations Duration 99", "group": "Meta", "used_for": "Used for monitoring the 99th percentile latency of TiDB metadata operations (such as schema changes, table creation, and DDL operations). Critical for diagnosing slow DDL execution and identifying performance bottlenecks in metadata management that could impact database operations.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 237, "dashboard": "TiDB-Cluster-Clinic", "title": "Historical Stats OPM", "group": "Statistics", "used_for": "Used for monitoring the operations per minute (OPM) of TiDB's historical statistics collection, tracking both statistics generation and dump operations with their success/failure rates. Essential for diagnosing issues with automatic statistics updates and identifying failures in the statistics maintenance process that could lead to suboptimal query plans.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[2m])) by (result)", "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[2m])) by (result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4025, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file discardable ratio distribution", "group": "Titan - $titan_db", "used_for": "Used for monitoring the distribution of discardable data ratios across Titan blob files in TiKV's storage engine. Helps identify garbage collection efficiency and determine when blob files contain high percentages of obsolete data that can be reclaimed through GC operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(tikv_engine_titandb_blob_file_discardable_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}) by (ratio)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 88, "dashboard": "TiDB-Cluster-Clinic", "title": "GC Failure OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) failures per minute in TiDB, categorized by failure type. Critical for identifying GC process issues that can lead to storage bloat, performance degradation, and potential cluster instability.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3414, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob key size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob keys in TiKV's Titan storage engine, showing average, P95, and P99 key sizes. Helps diagnose storage efficiency issues and identify abnormal key size patterns that may impact Titan's performance and space amplification.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_average\"})", "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile95\"})", "avg(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile99\"})", "max(tikv_engine_blob_key_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4023, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Iter touched blob file count", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the number of blob files accessed during iterator operations in TiKV's Titan storage engine. Helps diagnose Titan performance issues by identifying excessive blob file scanning, which can indicate fragmentation problems or inefficient range queries that require reading from multiple blob files.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_average\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile95\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile99\"})", "max(tikv_engine_blob_iter_touch_blob_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 163, "dashboard": "TiDB-Cluster-Clinic", "title": "Owner Watcher OPS", "group": "Owner", "used_for": "Used for monitoring the rate of owner watcher operations in TiDB's distributed coordination system. Helps diagnose issues with DDL owner election, schema synchronization, and cluster metadata management by tracking watcher operation throughput and success/failure patterns across instances.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_owner_watch_owner_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, result, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3655, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob get duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob retrieval operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately from the LSM tree, helping identify blob file access bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_get_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 86, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99", "group": "GC", "used_for": "Used for monitoring the 99th percentile latency of TiDB's garbage collection (GC) operations. Essential for identifying GC performance degradation that could impact transaction processing and detecting when old MVCC versions are taking too long to clean up.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 50, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID QPS", "group": "Meta", "used_for": "Used for monitoring the rate of auto-increment ID allocation operations per second in TiDB. Helps identify workload patterns involving auto-increment columns and diagnose potential bottlenecks in ID generation during high-throughput insert operations.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_autoid_operation_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 87, "dashboard": "TiDB-Cluster-Clinic", "title": "Config", "group": "GC", "used_for": "Used for monitoring TiDB's garbage collection (GC) configuration parameters across the cluster. Helps verify GC settings are properly configured and identify configuration inconsistencies that could impact storage reclamation and cluster performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4093, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob prev duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of backward iteration operations (prev) on blob files in TiKV's Titan storage engine. Critical for diagnosing performance issues with reverse scans over large values stored separately in Titan's blob storage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_prev_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 51, "dashboard": "TiDB-Cluster-Clinic", "title": "AutoID Duration", "group": "Meta", "used_for": "Used for monitoring the latency of TiDB's auto-increment ID allocation operations at p80 and p99 percentiles. Critical for diagnosing performance issues when applications experience delays in obtaining auto-increment IDs, particularly during high-concurrency insert workloads or when the auto-ID cache needs replenishment from PD.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 80, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 89, "dashboard": "TiDB-Cluster-Clinic", "title": "Action Result OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) action results per minute in TiDB, categorized by operation type. Helps diagnose GC efficiency issues, identify failed GC operations, and track the health of TiDB's distributed garbage collection process that cleans up obsolete MVCC versions.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_action_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3338, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob iter operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of iterator operations (seek, prev, next) on Titan blob files in TiKV's storage engine. Essential for diagnosing Titan blob storage I/O patterns and identifying potential performance issues when scanning or iterating over large values stored separately from the LSM tree.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_seek\"}[2m]))", "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_prev\"}[2m]))", "sum(rate(tikv_engine_blob_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_next\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 90, "dashboard": "TiDB-Cluster-Clinic", "title": "Too Many Locks Error OPM", "group": "GC", "used_for": "Used for monitoring the rate of \"too many locks\" errors encountered during garbage collection operations in TiDB. This panel helps identify when GC processes are blocked by excessive locks, which can indicate long-running transactions or heavy write workloads that prevent timely data cleanup.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_region_too_many_locks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 85, "dashboard": "TiDB-Cluster-Clinic", "title": "Worker Action OPM", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection (GC) worker operations per minute in TiDB, broken down by action type. Essential for diagnosing GC performance issues, identifying bottlenecks in the cleanup process, and ensuring efficient reclamation of obsolete MVCC versions.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3651, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync operations", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file sync operations in TiKV's Titan storage engine. Essential for diagnosing I/O performance issues and understanding the frequency of data persistence operations when using Titan for large value storage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Failure OPM", "group": "GC", "used_for": "Used for monitoring the per-minute failure rate of TiDB's unsafe destroy range operations during garbage collection. Critical for identifying issues with physical data deletion in TiKV, which can lead to disk space not being reclaimed properly or GC process failures.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 181, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Range Task Status", "group": "GC", "used_for": "Used for monitoring the status and results of delete range tasks in TiDB's garbage collection process. Helps diagnose issues with space reclamation after table/index drops, identifying failed or stuck delete operations that could lead to storage bloat.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3657, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file read duration", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring read latency from Titan blob files in TiKV's storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately in Titan's blob storage, helping identify I/O bottlenecks or storage degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_average\"})", "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile99\"})", "avg(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile95\"})", "max(tikv_engine_blob_file_read_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3643, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob bytes flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of data flow in TiKV's Titan storage engine, tracking bytes written to and read from blob files. Essential for understanding Titan's I/O patterns and diagnosing performance issues related to large value storage separation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 182, "dashboard": "TiDB-Cluster-Clinic", "title": "Push Task Duration 95", "group": "GC", "used_for": "Used for monitoring the 95th percentile latency of pushing range tasks from TiDB to TiKV during garbage collection operations. Critical for diagnosing GC performance bottlenecks and identifying instances experiencing delays in distributing GC workload to TiKV nodes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob next duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob iterator next operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when scanning large values stored separately in Titan's blob files, helping identify slow blob file reads that impact range query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_next_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3412, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob seek duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of seek operations in TiKV's Titan blob storage engine, showing average, P95, and P99 durations. Critical for diagnosing performance issues when reading large values stored separately in Titan's blob files, helping identify slow blob file access patterns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_seek_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3408, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file write duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring write latency to Titan blob files in TiKV's storage engine, showing average, P99, and P95 durations. Critical for diagnosing performance issues when large values are stored separately in Titan's blob storage, helping identify slow writes that could impact overall cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_average\"})", "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile99\"})", "avg(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile95\"})", "max(tikv_engine_blob_file_write_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3653, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob file sync duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the latency of blob file synchronization operations in TiKV's Titan storage engine, showing average, P95, and P99 durations. Critical for diagnosing Titan write performance issues and identifying when large value storage operations are experiencing slow disk sync times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_average\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile95\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile99\"})", "max(tikv_engine_blob_file_sync_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3645, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob keys flow", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob key operations in TiKV's Titan storage engine, tracking both keys written to and read from blob files. Essential for understanding Titan's key management patterns and diagnosing issues with large value storage when values exceed the blob file threshold.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 321, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing User Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of active user transactions in TiDB. Critical for identifying long-running transactions that may block garbage collection (GC) and cause performance degradation or storage bloat.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 203, "dashboard": "TiDB-Cluster-Clinic", "title": "No Available Connection Counter", "group": "Batch Client", "used_for": "Used for monitoring connection exhaustion in TiDB's batch client when no available connections to TiKV are found. Critical for diagnosing connection pool saturation issues that can cause query failures or timeouts when TiDB cannot establish connections to TiKV nodes.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tidb_tikvclient_batch_client_no_available_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3446, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob value size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size distribution of blob values stored in TiKV's Titan storage engine, showing average, P95, and P99 metrics. Essential for understanding large value storage patterns and optimizing Titan's blob file management when dealing with values exceeding the separation threshold.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_average\"})", "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile95\"})", "avg(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile99\"})", "max(tikv_engine_blob_value_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3649, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC keys flow", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the rate of key data flow during Titan blob garbage collection operations in TiKV. Helps diagnose Titan GC efficiency and identify potential issues with blob file cleanup that could impact storage space reclamation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5018, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC action", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of Titan blob garbage collection operations by action type in TiKV. Essential for tracking Titan's blob file cleanup efficiency and diagnosing storage space reclamation issues when using the Titan storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_action_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 204, "dashboard": "TiDB-Cluster-Clinic", "title": "Wait Connection Establish Duration", "group": "Batch Client", "used_for": "Used for monitoring the 99.99th percentile latency of establishing batch client connections between TiDB and TiKV. Critical for diagnosing connection pool exhaustion, network issues, or TiKV overload that causes extreme delays in connection establishment.", "component": "TIDB", "phenomenon": "latency", "object": "connection", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 180, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Client Unavailable Duration 95", "group": "Batch Client", "used_for": "Used for monitoring the 95th percentile duration when TiKV batch clients are unavailable in TiDB instances. Critical for diagnosing connection issues between TiDB and TiKV, helping identify network problems or TiKV unavailability that impacts query processing.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3340, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC bytes flow", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the byte flow rate during Titan blob garbage collection operations in TiKV. Helps identify GC throughput patterns and diagnose potential issues with Titan's large value storage cleanup process, which is critical for managing disk space and maintaining storage engine performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4021, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC input file size", "group": "Titan - $titan_db", "used_for": "Used for monitoring the size of input files during Titan blob garbage collection operations in TiKV. Helps diagnose Titan storage engine GC efficiency and identify potential issues with blob file accumulation by tracking average, P95, and P99 input file sizes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_average\"})", "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile95\"})", "avg(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile99\"})", "max(tikv_engine_blob_gc_input_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 228, "dashboard": "TiDB-Cluster-Clinic", "title": "Batch Receive Average Duration", "group": "Batch Client", "used_for": "Used for monitoring the average and P99 latency of batch receive operations between TiDB and TiKV clients. Critical for diagnosing network communication delays and identifying instances with slow batch RPC response times that could impact overall query performance.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_batch_recv_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_tikvclient_batch_recv_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4022, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC output file size", "group": "Titan - $titan_db", "used_for": "**Used For:**\n\nUsed for monitoring the size of output files generated during Titan blob garbage collection in TiKV. Helps assess GC efficiency and identify potential issues with blob file compaction, where unusually large or small output files may indicate suboptimal storage reclamation or fragmentation problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_average\"})", "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile95\"})", "avg(tikv_engine_blob_gc_output_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile99\"})", "max(tikv_engine_blob_gc_outputt_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2991, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Thread CPU", "group": "Pessimistic Locking", "used_for": "Used for monitoring CPU usage of TiKV's lock manager threads, including waiter manager and deadlock detector threads. Essential for diagnosing performance issues in pessimistic locking scenarios, particularly when experiencing lock contention or deadlock detection overhead.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"waiter_manager.*\"}[2m])) by (instance, name)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"deadlock_detect.*\"}[2m])) by (instance, name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3344, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC file count", "group": "Titan - $titan_db", "used_for": "Used for monitoring the rate of blob file garbage collection operations in TiKV's Titan storage engine. Helps diagnose Titan GC efficiency and identify potential issues with blob file cleanup that could lead to disk space accumulation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "sum(rate(tikv_engine_blob_gc_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 309, "dashboard": "TiDB-Cluster-Clinic", "title": "Ongoing Internal Transaction Duration", "group": "GC", "used_for": "Used for monitoring the 99th percentile duration of TiDB's internal transactions, such as those used for GC (garbage collection) and statistics updates. Critical for diagnosing when internal maintenance operations are taking too long and potentially impacting cluster performance or causing transaction conflicts.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_executor_ongoing_txn_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"internal\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 241, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Profiling OPS", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CPU profiling operations in TiDB instances. Helps verify that CPU profiling is actively collecting performance data for TopSQL analysis and troubleshooting query performance issues.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_cpu_profile_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4018, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Waiting Queue", "group": "Pessimistic Locking", "used_for": "Used for monitoring the size and status of pessimistic lock waiting queues in TiKV. Essential for diagnosing lock contention issues, identifying transactions blocked by lock conflicts, and detecting potential deadlock situations that impact transaction throughput.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(tikv_lock_manager_wait_table_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)", "sum(max_over_time(tikv_lock_wait_queue_entries_gauge_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4019, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detector leader", "group": "Pessimistic Locking", "used_for": "Used for identifying which TiKV instance is currently serving as the deadlock detector leader in pessimistic locking scenarios. Essential for troubleshooting deadlock detection issues and verifying proper leader election when investigating transaction conflicts or lock-related problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[15s])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2995, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Deadlock detect duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring the latency of deadlock detection in TiKV's pessimistic locking mechanism, showing both average and p99 durations. Critical for diagnosing transaction contention issues and identifying when deadlock detection becomes a performance bottleneck in high-concurrency pessimistic transaction workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_detect_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_detect_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572093, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total pessimistic locks memory size", "group": "Pessimistic Locking", "used_for": "Used for monitoring the total memory consumed by pessimistic locks across TiKV instances. Critical for diagnosing memory pressure issues caused by long-running transactions or lock accumulation, and helps identify potential out-of-memory risks in pessimistic transaction scenarios.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "tidb_cluster" ], "promql": [ "tikv_pessimistic_lock_memory_size{tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 242, "dashboard": "TiDB-Cluster-Clinic", "title": "Report Data Count Per Minute", "group": "TopSQL", "used_for": "Used for monitoring the volume of Top SQL data being reported by TiDB instances per minute, broken down by data type. Helps diagnose Top SQL feature health and identify potential issues with SQL performance data collection or reporting bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_report_data_total_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, job)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 298, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of rows inserted by TTL (Time To Live) operations per hour in TiDB. Helps track TTL job activity and identify unusual insertion patterns or performance issues related to TTL background tasks.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1h]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 243, "dashboard": "TiDB-Cluster-Clinic", "title": "Total Report Count", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for monitoring the total number of TopSQL reports sent from TiDB and TiKV instances, broken down by report type and result status. Essential for verifying that TopSQL data collection is functioning correctly and diagnosing issues when query performance analysis data is missing or incomplete.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, result)", "sum(increase(tidb_topsql_report_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, result)", "sum(tikv_resource_metering_report_data_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"sent\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Detect error", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of errors occurring in TiKV's pessimistic lock manager, categorized by error type. Essential for diagnosing locking failures, deadlocks, and other concurrency control issues that may impact transaction processing and data consistency.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573091, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pessimistic lock activities", "group": "Pessimistic Locking", "used_for": "Used for monitoring the number of transactions waiting in pessimistic lock queues, categorized by lock type. Essential for diagnosing lock contention issues, identifying bottlenecks in concurrent transaction processing, and understanding when transactions are blocked waiting for locks to be released.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_lock_wait_queue_entries_gauge_vec{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 299, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV CPU", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiKV instances in the context of TTL (Time To Live) operations. Helps identify if TTL background tasks are causing excessive CPU consumption on specific TiKV nodes, which could impact overall cluster performance.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\".*tikv\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 239, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Report Duration", "group": "TopSQL", "used_for": "Used for monitoring the 99th percentile latency of TopSQL report generation and submission in TiDB and TiKV. Critical for diagnosing performance issues with the TopSQL feature itself, helping identify when report collection overhead becomes excessive or when communication delays occur between components.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"ok\"}[5m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lengths of lock wait queues when transaction enqueues", "group": "Pessimistic Locking", "used_for": "Used for monitoring the distribution of lock wait queue lengths when transactions enqueue for pessimistic locks in TiKV. Helps diagnose lock contention issues and identify when multiple transactions are waiting for the same resources, which can indicate hotspot problems or inefficient transaction patterns.", "component": "TIKV", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_lock_wait_queue_length_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL QPS By Type", "group": "TTL", "used_for": "Used for monitoring the queries per second (QPS) of TTL (Time To Live) operations in TiDB, broken down by SQL type and execution result. Helps identify TTL workload patterns, track the volume of automatic data cleanup operations, and diagnose issues when TTL jobs are running slower than expected or failing.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_query_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 287, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Processed Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows processed by TiDB's Time-To-Live (TTL) feature, broken down by SQL operation type and result status. Essential for tracking TTL job performance, identifying processing bottlenecks, and ensuring expired data is being cleaned up at the expected rate.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573558, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read File into Memory Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the time taken to read and download backup files into memory during Point-in-Time Restore operations in TiKV. Critical for diagnosing slow restore performance and identifying bottlenecks in the file download phase of the restore process.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_apply_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"exec_download\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573350, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring CPU usage of SST (Sorted String Table) file processing threads in TiKV during Point In Time Restore operations. Helps identify CPU bottlenecks and resource contention when restoring data from backups, ensuring restore performance meets expectations.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573352, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of import RPC operations during Point In Time Restore (PITR), tracking both per-instance apply requests and cluster-wide import operations. Essential for diagnosing restore performance issues and identifying bottlenecks in the data import process during recovery operations.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[$__rate_interval])) by (instance, request)", "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (request)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 305, "dashboard": "TiDB-Cluster-Clinic", "title": "Ignore Event Per Minute", "group": "TopSQL", "used_for": "**Used For:**\n\nUsed for tracking the rate of SQL statements ignored by TopSQL feature per minute, categorized by ignore reason type. Helps diagnose why certain queries are not being captured in TopSQL statistics, which is essential for troubleshooting incomplete query performance monitoring and understanding TopSQL sampling limitations.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_topsql_ignored_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 302, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert Rows Per Second", "group": "TTL", "used_for": "Used for monitoring the rate of rows inserted per second by TiDB's Time-To-Live (TTL) feature. Helps identify TTL job activity levels and diagnose potential performance impacts when TTL operations are inserting data at high rates.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 281, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Job Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) jobs across different statuses in TiDB. Helps identify job execution patterns, detect stuck or failed TTL jobs, and diagnose issues with automatic data cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_job_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 276, "dashboard": "TiDB-Cluster-Clinic", "title": "Scan Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) scan workers in TiDB. Helps identify which phase (scanning, deleting, or waiting) is consuming the most time, enabling diagnosis of TTL job performance bottlenecks and optimization opportunities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan_worker\"}[2m])) by (phase)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 303, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Rows Per Hour", "group": "TTL", "used_for": "Used for monitoring the number of expired rows successfully deleted by TTL (Time To Live) jobs per hour in TiDB. Helps track TTL cleanup efficiency and identify potential issues with automatic data expiration and deletion processes.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573349, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Throughput", "group": "Point In Time Restore", "used_for": "Used for monitoring the throughput of apply requests during Point In Time Restore (PITR) operations in TiKV, measured in bytes per second. Essential for tracking restore performance and identifying bottlenecks when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_apply_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 245, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV Stat Task OPS", "group": "TopSQL", "used_for": "Used for monitoring the rate of resource metering statistics tasks processed by TiKV instances. Helps diagnose TopSQL feature performance and identify issues with resource consumption tracking when analyzing query-level resource usage patterns.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_resource_metering_stat_task_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 285, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Delete Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution of TTL (Time To Live) delete queries in TiDB across multiple percentiles (p50, p80, p90, p99). Essential for diagnosing performance issues with automatic data expiration operations and identifying when TTL delete queries are experiencing slowdowns that could impact overall cluster performance.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Waiter lifetime duration", "group": "Pessimistic Locking", "used_for": "Used for monitoring how long transactions wait for locks to be released in TiKV's pessimistic locking system, tracking average, p99, and p9999 latencies. Critical for diagnosing lock contention issues and identifying transactions experiencing excessive wait times due to conflicting pessimistic locks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573351, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "P99 RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the 99th percentile RPC latency during Point In Time Restore (PITR) operations in TiKV, including apply requests, download queue/execution times, and engine operations. Critical for diagnosing performance bottlenecks and slow restore operations when recovering data to a specific point in time.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[2m])) by (le, request))", "histogram_quantile(0.99, avg(rate(tikv_import_apply_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"queue|exec_download\"}[2m])) by (le, type))", "histogram_quantile(0.99, avg(rate(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 282, "dashboard": "TiDB-Cluster-Clinic", "title": "Delete Worker Time By Phase", "group": "TTL", "used_for": "Used for monitoring the time distribution across different phases of TTL (Time To Live) delete workers in TiDB. Helps identify performance bottlenecks in the TTL deletion process by showing which phase (e.g., scanning, deleting, waiting) consumes the most time.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_ttl_phase_time{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"delete_worker\"}[2m])) by (phase)\n" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572094, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "In-memory pessimistic locking result", "group": "Pessimistic Locking", "used_for": "Used for monitoring the success and failure rates of in-memory pessimistic lock operations in TiKV. Helps diagnose lock contention issues and memory-based pessimistic locking performance, particularly useful when investigating transaction conflicts or lock acquisition failures that don't require disk persistence.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_in_memory_pessimistic_locking{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 297, "dashboard": "TiDB-Cluster-Clinic", "title": "TiDB CPU Usage", "group": "TTL", "used_for": "Used for monitoring CPU utilization of TiDB server instances, showing actual CPU usage rate against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks and resource saturation issues in TiDB nodes, particularly when investigating TTL (Time To Live) job performance impacts.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 284, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Scan Query Duration", "group": "TTL", "used_for": "Used for monitoring the latency distribution (p50, p80, p90, p99) of TTL scan SELECT queries in TiDB. Essential for diagnosing performance issues with Time-To-Live background scanning operations and identifying when expired data cleanup is impacting query response times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))\n", "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"select\", result=\"ok\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 301, "dashboard": "TiDB-Cluster-Clinic", "title": "TiKV IO MBps", "group": "TTL", "used_for": "Used for monitoring TiKV's I/O throughput in megabytes per second during TTL (Time To Live) operations, tracking WAL writes and data reads across instances. Essential for identifying I/O bottlenecks and uneven load distribution when TTL processes are deleting expired data, helping diagnose performance degradation caused by excessive disk I/O.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))", "max(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[2m])) by (instance))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573229, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Queuing Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of queuing time for import engine requests in TiKV during Point In Time Restore operations. Helps identify bottlenecks and delays in the restore pipeline when requests are waiting to be processed, which is critical for diagnosing slow backup restoration performance.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queuing\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573032, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cache Events", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of cache events during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose cache behavior and performance during data import and restore processes, identifying potential bottlenecks in cache hit/miss patterns.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_apply_cache_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573348, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Overall RPC Duration", "group": "Point In Time Restore", "used_for": "Used for monitoring the overall RPC duration distribution for Point-in-Time Restore (PITR) apply operations across TiKV instances. Essential for diagnosing performance bottlenecks and latency issues during backup restoration processes, helping identify slow apply operations that may impact recovery time objectives.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=\"apply\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 294, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Task Count By Status", "group": "TTL", "used_for": "Used for monitoring the distribution of TTL (Time To Live) tasks across different statuses in TiDB. Helps track task execution states (running, completed, failed) to identify TTL job bottlenecks or failures when automatic data cleanup is not working as expected.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(tidb_server_ttl_task_status{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573344, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Downloaded File Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of file sizes downloaded during Point-in-Time Restore (PITR) operations in TiKV. Helps diagnose restore performance issues by tracking download bandwidth usage and identifying whether large file transfers are causing bottlenecks during backup restoration.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_download_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2877, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock Manager Handled tasks", "group": "Pessimistic Locking", "used_for": "Used for monitoring the rate of different task types handled by TiKV's lock manager in pessimistic transaction mode. Essential for diagnosing lock contention issues, deadlock detection performance, and understanding the distribution of pessimistic locking operations across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_lock_manager_task_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573230, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blocked by Concurrency Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time that Point In Time Restore (PITR) operations spend waiting for concurrency permits in TiKV's import engine. Helps diagnose restore performance bottlenecks caused by concurrency limitations when multiple restore tasks compete for resources.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"get_permit\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 300, "dashboard": "TiDB-Cluster-Clinic", "title": "Table Count By TTL Schedule Delay", "group": "TTL", "used_for": "Used for monitoring the number of tables grouped by their TTL (Time To Live) scheduling delay in TiDB. Helps identify tables experiencing delayed TTL job execution, which is critical for diagnosing data cleanup performance issues and ensuring timely expiration of time-sensitive data.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tidb_server_ttl_watermark_delay{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", type=\"schedule\"}) by (type, name)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573233, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Batch Size", "group": "Point In Time Restore", "used_for": "Used for monitoring the distribution of batch sizes during Point-in-Time Restore (PITR) apply operations in TiKV. Helps diagnose restore performance by showing how data is being applied in batches, which is critical for identifying inefficient batch processing or bottlenecks during recovery operations.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_apply_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12842, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write Pipeline Duration", "group": "Duration", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile latency of TiKV's write pipeline stages, including Raft log append duration, request wait time, and apply wait time. Critical for diagnosing write performance bottlenecks and identifying which stage of the write path is causing slowdowns in data persistence operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_msg_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573346, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cached File in Memory", "group": "Point In Time Restore", "used_for": "Used for monitoring the amount of cached file data in memory during Point In Time Restore (PITR) operations in TiKV. Helps diagnose memory usage patterns and potential memory pressure issues when restoring data from backups.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_import_apply_cached_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 304, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Insert/Delete Rows By Day", "group": "TTL", "used_for": "Used for monitoring the daily volume of TTL (Time To Live) operations, tracking both rows inserted into TTL tables and expired rows successfully deleted. Essential for understanding TTL workload patterns, verifying automatic data cleanup is functioning correctly, and identifying potential issues with expired data accumulation or deletion throughput.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[${__to:date:H}h${__to:date:m}m]))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset ${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_insert_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))", "sum(increase(tidb_server_ttl_processed_expired_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"delete\", result=\"ok\"}[1d] offset 1d${__to:date:H}h${__to:date:m}m))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573118, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Request Speed", "group": "Point In Time Restore", "used_for": "Used for monitoring the rate of Point-in-Time Restore (PITR) apply requests being initiated in TiKV. Helps track the speed of data restoration operations and diagnose slow recovery performance during point-in-time restore procedures.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_applier_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"begin_req\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573119, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Engine Requests Unfinished", "group": "Point In Time Restore", "used_for": "Used for monitoring the number of incomplete engine import requests in TiKV during Point In Time Restore operations. Helps identify stuck or slow import processes that may be blocking restore progress or causing recovery delays.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "increase(tikv_import_applier_event{instance=~\"$instance\", type!=\"begin_req\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1707, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Capacity size", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity configured for each TiKV instance. Essential for capacity planning and identifying instances approaching their storage limits before they run out of disk space.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"capacity\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1706, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Available size", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the available disk space on each TiKV instance. Critical for preventing storage exhaustion issues and ensuring sufficient capacity for data writes and compaction operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"available\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 313, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Event Count Per Minute", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) events occurring per minute in TiDB, broken down by event type. Essential for tracking TTL job activity, identifying abnormal deletion patterns, and diagnosing issues with automatic data expiration and cleanup operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_ttl_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=\"hook.tidb.ttl\"}[1m])) by (type)", "sum(increase(tidb_server_timer_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"runtime\\\\.ttl.*\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 293, "dashboard": "TiDB-Cluster-Clinic", "title": "EMA CPU Usage", "group": "Resource Manager", "used_for": "Used for monitoring the Exponential Moving Average (EMA) of CPU usage by TiDB's Resource Manager. Helps track smoothed CPU consumption trends to identify resource allocation patterns and detect sustained high CPU usage that may require resource group adjustments.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_rm_ema_cpu_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12970, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Cop Read Duration", "group": "Duration", "used_for": "**Used For:**\n\nUsed for monitoring the end-to-end latency of coprocessor read requests in TiKV at p95 and p99 percentiles, including snapshot acquisition, request queuing, and processing time. Critical for diagnosing slow analytical queries and identifying whether performance bottlenecks occur during storage access, request scheduling, or coprocessor execution phases.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 289, "dashboard": "TiDB-Cluster-Clinic", "title": "GOGC", "group": "Resource Manager", "used_for": "Used for monitoring the GOGC (Go Garbage Collection) parameter value in TiDB instances. Helps diagnose memory management behavior and understand GC tuning configurations that affect TiDB's memory usage patterns and performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_gogc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763573449, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Store Memory Usage", "group": "Point In Time Restore", "used_for": "Used for monitoring memory consumption of TiKV's Raft Store component, which handles consensus protocol operations and log replication. Critical for diagnosing memory pressure issues during Point In Time Restore operations and identifying instances with abnormal Raft-related memory usage.", "component": "TIKV", "phenomenon": "memory", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573231, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply Time", "group": "Point In Time Restore", "used_for": "Used for monitoring the time distribution of applying SST files during Point-in-Time Restore operations in TiKV. Helps diagnose performance bottlenecks and slow apply operations when restoring data to a specific point in time.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tikv_import_engine_request_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[$__rate_interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 56, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store size", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring storage space utilization across TiKV instances, breaking down total used space into KV data and Raft log sizes. Essential for capacity planning, identifying storage imbalances between nodes, and detecting potential disk space issues before they impact cluster operations.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"used\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"kv_size\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"raft_size\"}) by (instance)", "sum(tikv_store_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"import_size\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1708, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU", "group": "", "used_for": "Used for monitoring CPU utilization of individual TiKV instances in the cluster. Essential for identifying CPU bottlenecks, unbalanced load distribution across nodes, and determining if TiKV instances are CPU-saturated during performance issues.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\".*tikv\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8385, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolved TS Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's Resolved TS worker threads, which maintain transaction timestamp consistency for CDC and other features. Essential for diagnosing performance bottlenecks in change data capture pipelines and identifying instances where Resolved TS processing is consuming excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"resolved_ts.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1713, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "QPS", "group": "", "used_for": "Used for monitoring the queries per second (QPS) handled by each TiKV instance, broken down by gRPC message type. Essential for identifying which TiKV nodes are handling the most traffic and detecting imbalanced workload distribution or sudden throughput changes across the cluster.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance,type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9162, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advance ts Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's advance-ts worker threads that drive the Resolved TS mechanism forward. Essential for diagnosing performance bottlenecks in CDC (Change Data Capture) and follower read scenarios where Resolved TS advancement may be slow or consuming excessive CPU resources.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"advance_ts.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 310, "dashboard": "TiDB-Cluster-Clinic", "title": "Closest Replica Hit Count Per Second", "group": "Follower Read", "used_for": "Used for monitoring the rate of queries that successfully hit the closest replica when follower read is enabled in TiDB. Helps verify follower read effectiveness and diagnose whether queries are properly utilizing nearby replicas to reduce cross-AZ network latency.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_closest_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9164, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scan lock Worker CPU", "group": "Resolved-TS", "used_for": "Used for monitoring CPU usage of TiKV's incremental scan lock workers that support Resolved-TS functionality. Essential for diagnosing performance issues in CDC (Change Data Capture) and transaction consistency features, particularly when experiencing high CPU consumption or slow data replication.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"inc_scan.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8387, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum gap between resolved timestamps and current time in TiKV instances. Critical for diagnosing CDC (Change Data Capture) replication lag and identifying instances where transaction resolution is delayed, which can impact downstream data synchronization and consistency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1711, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "MBps", "group": "", "used_for": "Used for monitoring the read and write throughput (in megabytes per second) of TiKV's RocksDB storage engine. Essential for identifying I/O bottlenecks, tracking WAL write rates, and diagnosing storage performance issues when query latency increases or disk utilization spikes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[2m])) by (instance)", "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1709, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory", "group": "", "used_for": "Used for monitoring the resident memory usage of each TiKV instance in the cluster. Essential for detecting memory leaks, identifying instances approaching OOM conditions, and diagnosing memory-related performance degradation or stability issues.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tikv\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572078, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Resolved TS Region", "group": "Resolved-TS", "used_for": "Used for monitoring the region with the minimum resolved timestamp on each TiKV instance. Critical for diagnosing Resolved TS lag issues and identifying which specific region is blocking transaction consistency advancement in CDC or follower read scenarios.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_resolved_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1710, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO utilization", "group": "", "used_for": "Used for monitoring the percentage of time that disk I/O operations are in progress on TiKV nodes. Critical for identifying disk saturation and I/O bottlenecks that can severely impact TiKV read/write performance and overall cluster stability.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 318, "dashboard": "TiDB-Cluster-Clinic", "title": "Delivered kv count", "group": "Import Into", "used_for": "Used for monitoring the total number of key-value pairs delivered during IMPORT INTO operations in TiDB. Helps track data ingestion progress and diagnose slow import performance by identifying bottlenecks in the KV delivery pipeline.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_block_deliver_kv_pairs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 324, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size Per Instance and TiKV", "group": "Follower Read", "used_for": "Used for monitoring the rate of coprocessor response data size from each TiKV store to TiDB instances during follower read operations. Helps identify network bandwidth usage patterns and diagnose potential data transfer bottlenecks between TiDB and TiKV nodes.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 319, "dashboard": "TiDB-Cluster-Clinic", "title": "Data read/encode/deliver average duration", "group": "Import Into", "used_for": "Used for monitoring the average latency of three key stages in TiDB's IMPORT INTO operation: reading source data, encoding rows, and delivering data blocks. Essential for diagnosing performance bottlenecks during bulk data import operations and identifying which stage is causing slowdowns.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_import_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tidb_import_row_read_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tidb_import_row_read_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1712, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errps", "group": "", "used_for": "Used for monitoring error rates per second across TiKV instances, tracking gRPC message failures, PD heartbeat anomalies (noop messages), and critical errors by type. Essential for quickly identifying unhealthy TiKV nodes and diagnosing communication or internal failures that could impact cluster stability and data availability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)", "sum(delta(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"noop\"}[2m])) by (instance) < 1", "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9168, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check leader duration", "group": "Resolved-TS", "used_for": "Used for monitoring the time spent checking Raft leader status during resolved timestamp advancement in TiKV. Critical for diagnosing resolved-ts lag issues caused by slow leader verification, which can impact CDC (Change Data Capture) and backup/restore operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 316, "dashboard": "TiDB-Cluster-Clinic", "title": "Total encode/deliver/import-kv speed", "group": "Import Into", "used_for": "Used for monitoring the throughput of IMPORT INTO operations across three stages: data encoding (restored), delivery to TiKV (written), and final import completion (imported). Essential for diagnosing performance bottlenecks during bulk data import tasks and identifying which stage is limiting overall import speed.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}[2m])) by(task_id)", "sum(rate(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}[2m])) by(task_id)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4106, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Uptime", "group": "", "used_for": "Used for monitoring how long each TiKV instance has been running since its last restart. Essential for identifying recent restarts, detecting instance instability, and verifying cluster availability during troubleshooting or after maintenance operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=~\".*tikv\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1714, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region", "group": "", "used_for": "Used for monitoring the total number of regions and buckets stored on each TiKV instance. Essential for identifying data distribution imbalances, region scheduling issues, and detecting instances that may be overloaded with too many regions, which can impact I/O performance and cluster stability.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"buckets\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 320, "dashboard": "TiDB-Cluster-Clinic", "title": "Import/Normal mode", "group": "Import Into", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration during data import operations. Helps verify if TiKV is in import mode (higher limit) versus normal mode (lower limit), which is critical for diagnosing import performance issues and ensuring proper mode transitions.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 325, "dashboard": "TiDB-Cluster-Clinic", "title": "Coprocessor Response Size", "group": "Follower Read", "used_for": "Used for monitoring the size of coprocessor responses returned from TiKV to TiDB, showing average, median (p50), p80, and p99 percentiles. Essential for diagnosing network bandwidth issues, identifying queries that transfer excessive data, and understanding the I/O characteristics of follower read operations.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_copr_resp_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_distsql_copr_resp_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.5, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.8, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_distsql_copr_resp_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572077, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of resolved-ts in region leaders", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum time gap between the current time and the resolved timestamp maintained by region leaders in TiKV. Critical for diagnosing replication lag issues in CDC (Change Data Capture) and identifying regions that are falling behind in maintaining consistent snapshots for downstream consumers.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573950, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scan backoff duration", "group": "Resolved-TS", "used_for": "Used for monitoring the backoff duration during initial scans in TiKV's Resolved TS mechanism. Helps diagnose delays in establishing resolved timestamps, which can indicate resource contention or performance issues affecting CDC (Change Data Capture) and other features relying on consistent snapshots.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573804, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Safe TS Follower Region", "group": "Resolved-TS", "used_for": "Used for monitoring which TiKV regions have the minimum safe timestamp among follower replicas in the Resolved TS mechanism. Critical for diagnosing stale read issues and identifying regions with lagging replication that may block CDC or backup progress.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_follower_safe_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1715, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader", "group": "", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance and detecting sudden leader drops. Critical for identifying leader balance issues, region scheduling problems, or potential TiKV node failures when leader count drops significantly (>10 regions in 2 minutes).", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1584, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiKV server overload conditions by tracking scheduler busy errors, channel full events, and coprocessor request rejections. Critical for diagnosing when TiKV instances are overwhelmed by excessive workload, helping identify capacity bottlenecks and the need for resource scaling or load balancing.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", "sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573805, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max gap of follower safe-ts", "group": "Resolved-TS", "used_for": "Used for monitoring the maximum time gap between follower replicas' safe timestamps in TiKV's Resolved TS mechanism. Critical for diagnosing replication lag issues and ensuring data consistency across followers, especially when troubleshooting stale read problems or CDC (Change Data Capture) delays.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_follower_safe_ts_gap_millis{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572000, "dashboard": "TiDB-Cluster-Clinic", "title": "Ingest Worker Count", "group": "Global Sort", "used_for": "Used for monitoring the number of active ingest workers handling global sort operations in TiDB. Helps diagnose performance bottlenecks and resource utilization issues during large-scale data import or DDL operations that require global sorting.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_global_sort_ingest_worker_cnt{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 317, "dashboard": "TiDB-Cluster-Clinic", "title": "Encoded/Delivered/Imported data size", "group": "Import Into", "used_for": "Used for monitoring the data flow during IMPORT INTO operations in TiDB, tracking three key stages: encoded data size (total_restore), data written to storage (written), and data successfully restored (restored). Essential for diagnosing import performance bottlenecks, verifying data processing progress, and identifying discrepancies between encoded, written, and imported data volumes.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"restored\"}", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"written\"}", "sum(tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"total_restore\"}) by(task_id)", "tidb_import_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", state=\"imported\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1718, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiKV's Raftstore engine, broken down by instance and error status. Critical for detecting storage layer issues, consensus failures, or I/O problems that could impact data availability and cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2741, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiKV instances, categorized by error type. Essential for detecting severe system issues like data corruption, disk failures, or internal panics that require immediate attention and may impact cluster stability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 18, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiKV instances, categorized by failure type and store. Critical for identifying communication issues, heartbeat failures, or store unavailability problems that can impact cluster stability and cause request latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572079, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Min Leader Resolved TS Region", "group": "Resolved-TS", "used_for": "Used for monitoring the region with the minimum resolved timestamp among all leader regions on each TiKV instance. Critical for identifying lagging regions that may block CDC (Change Data Capture) or stale read operations, helping diagnose data replication delays and transaction visibility issues.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_region{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571998, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Duration", "group": "Global Sort", "used_for": "Used for monitoring the latency of reading data from cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that rely on cloud storage, particularly when investigating slow bulk data operations or import jobs.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763571996, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Duration", "group": "Global Sort", "used_for": "**Used For:**\n\nUsed for monitoring the latency of writing data to cloud storage during TiDB's global sort operations, showing p50, p95, and p999 percentiles. Critical for diagnosing performance bottlenecks in distributed sorting tasks that involve cloud storage, particularly when investigating slow data import or DDL operations using global sort.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8379, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Lock heap size", "group": "Resolved-TS", "used_for": "Used for monitoring the memory consumption of the lock heap in TiKV's Resolved TS mechanism. Helps diagnose memory pressure issues and identify instances with excessive lock accumulation that may impact transaction processing and CDC (Change Data Capture) performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_resolved_ts_lock_heap_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8377, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observe region status", "group": "Resolved-TS", "used_for": "Used for monitoring the distribution of region resolution statuses in TiKV's Resolved-TS mechanism. Helps diagnose issues with transaction consistency and CDC (Change Data Capture) by tracking which regions are successfully resolving timestamps versus those experiencing delays or failures.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_resolved_ts_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571999, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the read throughput rate from cloud storage during TiDB's global sort operations at different percentiles (p50, p95, p999). Essential for diagnosing performance bottlenecks in cloud-based data sorting tasks and identifying when cloud storage I/O becomes a limiting factor for large-scale data operations like index creation or data import.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_read_from_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 1719, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring the rate of scheduler errors in TiKV, specifically tracking snapshot and prepare write failures. Critical for diagnosing transaction processing issues and identifying instances experiencing storage layer problems that could impact data consistency or write operations.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571997, "dashboard": "TiDB-Cluster-Clinic", "title": "Write To Cloud Storage Rate", "group": "Global Sort", "used_for": "Used for monitoring the write throughput rate (at p50, p95, and p999 percentiles) when TiDB's Global Sort feature writes data to cloud storage during distributed operations like `ADD INDEX`. Critical for diagnosing performance bottlenecks in cloud storage write operations and identifying whether slow index creation or data import tasks are caused by cloud storage write latency.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_global_sort_write_to_cloud_storage_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9166, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fail advance ts count", "group": "Resolved-TS", "used_for": "Used for monitoring failures in advancing the resolved timestamp in TiKV, which is critical for CDC (Change Data Capture) and stale read functionality. Helps diagnose issues with transaction consistency guarantees and identify reasons why resolved timestamps cannot progress, such as stale peer checks or region problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", "sum(delta(tikv_raftstore_check_stale_peer{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1722, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region transfers or drops on each TiKV instance. Critical for detecting leadership instability, network partitions, or node failures that cause frequent leader changes and potential service disruptions.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "Dataplane Resource Overview", "title": "Pod Restart Count", "group": "", "used_for": "Used for tracking the cumulative restart count of pods in the dataplane namespace. Essential for identifying unstable pods experiencing crashes or OOM kills, which can indicate resource constraints, configuration issues, or application bugs affecting cluster stability.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "cluster", "control_plane_info", "namespace" ], "promql": [ "sum(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum(kube_pod_container_status_restarts_total{control_plane_info=\"$control_plane_info\",cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 8383, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request size", "group": "Resolved-TS", "used_for": "Used for monitoring the 99th percentile size of CheckLeader requests in TiKV's Resolved-TS mechanism, tracking both byte size and item count. Essential for diagnosing Resolved-TS performance issues and identifying instances with abnormally large CheckLeader requests that could impact transaction consistency guarantees.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8381, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending command size", "group": "Resolved-TS", "used_for": "Used for monitoring the total size of pending commands in TiKV's Resolved-TS channel. Helps identify memory pressure and potential bottlenecks when commands accumulate faster than they can be processed, which may indicate Resolved-TS lag or replication issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_resolved_ts_channel_penging_cmd_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1721, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV instances, broken down by message type. Critical for identifying communication failures between TiKV and other components (TiDB, PD, or other TiKV nodes), which can indicate network issues, resource exhaustion, or service degradation.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "Dataplane Resource Overview", "title": "CPU Throttle", "group": "", "used_for": "Used for monitoring CPU throttling rates of containers in the dataplane namespace. Helps identify when pods are hitting CPU limits and experiencing performance degradation due to resource constraints, indicating potential need for CPU quota adjustments.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod, namespace)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 1720, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiKV instances, broken down by error reason. Critical for diagnosing issues with pushed-down query execution, such as region errors, lock conflicts, or timeout problems that affect query performance and reliability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12308, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% CheckLeader request region count", "group": "Resolved-TS", "used_for": "Used for monitoring the 99th percentile number of regions checked per CheckLeader request in TiKV's Resolved-TS mechanism. Helps identify potential performance bottlenecks when a single CheckLeader request needs to verify an unusually high number of regions, which can impact transaction consistency checks and CDC (Change Data Capture) performance.", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573729, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Allocator Stats", "group": "Memory", "used_for": "Used for monitoring memory allocator statistics in TiKV instances, tracking memory allocation patterns and usage across different allocator components. Essential for diagnosing memory-related issues, memory leaks, and understanding TiKV's memory consumption behavior at the allocator level.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_allocator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1723, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions missing their leader on each TiKV instance. Critical for detecting split-brain scenarios, network partitions, or leader election failures that can cause service unavailability and data access issues.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573730, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "group": "Memory", "used_for": "Used for monitoring the net memory allocation rate per thread in TiKV, showing allocated bytes (positive) minus released bytes (negative). Essential for identifying memory leaks, understanding per-thread memory pressure, and diagnosing which threads are accumulating memory over time.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name) - sum(rate(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"dealloc\"}[$__rate_interval])) by (thread_name) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3638, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes across TiKV instances to understand data distribution patterns. Helps identify region size imbalances, oversized regions that may need splitting, or undersized regions that could indicate excessive splitting, which can impact I/O performance and cluster efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 42, "dashboard": "Dataplane Resource Overview", "title": "Pod Abnormal Phase", "group": "", "used_for": "Used for identifying pods in abnormal states (not Running or Succeeded) within the specified namespace. Critical for quickly detecting pod failures, crashes, or scheduling issues that require immediate attention in the dataplane cluster.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\",namespace=\"$namespace\",phase!~\"Running|Succeeded\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572510, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Damaged files", "group": "Errors", "used_for": "Used for monitoring corrupted files detected in TiKV's RocksDB storage engine and tracking how many damaged files have been deleted. Critical for identifying data integrity issues and potential disk corruption that could lead to data loss or cluster instability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_rocksdb_damaged_files{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "tikv_rocksdb_damaged_files_deleted{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 38, "dashboard": "Dataplane Resource Overview", "title": "Pod Last Terminated Reason", "group": "", "used_for": "Used for tracking why pods in the dataplane namespace were last terminated, excluding normal completions. Essential for diagnosing pod crashes, OOM kills, and other abnormal termination events that may indicate resource constraints or application failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", reason!=\"Completed\"}) by (pod,reason)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 2696, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Newly Allocated Bytes by Thread", "group": "Memory", "used_for": "Used for tracking memory allocation rates across different TiKV threads. Helps identify which threads are consuming the most memory and diagnose memory-related performance issues or potential memory leaks in specific thread operations.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572588, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Log Replication Rejected", "group": "Errors", "used_for": "Used for monitoring the rate of rejected Raft log append operations in TiKV instances. Critical for identifying replication bottlenecks, follower lag issues, or resource constraints that prevent nodes from accepting log entries from the Raft leader.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 26, "dashboard": "Dataplane Resource Overview", "title": "Node Load 1m", "group": "Node", "used_for": "Used for monitoring the 1-minute load average per CPU core across cluster nodes. Helps identify nodes experiencing high system load or resource contention, which can indicate performance bottlenecks or capacity issues requiring investigation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_load1_per_cpu:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 2, "dashboard": "Dataplane Resource Overview", "title": "Nodes", "group": "Node", "used_for": "Used for monitoring the distribution of Kubernetes nodes by instance type in the cluster. Helps identify node composition, capacity planning needs, and potential resource imbalances across different machine types.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(kube_node_labels{cluster=\"$cluster\"}) by (label_beta_kubernetes_io_instance_type)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 71, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested CPU resources to allocatable CPU capacity on each Kubernetes node. Helps identify nodes approaching resource limits and potential scheduling constraints that could prevent new pod deployments.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763573731, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Recently Released Bytes by Thread", "group": "Memory", "used_for": "Used for tracking memory deallocation patterns across different TiKV threads. Helps identify which threads are releasing memory and diagnose potential memory leaks or inefficient memory management when deallocation rates are abnormally low or imbalanced.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_allocator_thread_allocation{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"alloc\"}[$__rate_interval])) by (thread_name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "Dataplane Resource Overview", "title": "Node CPU Usage", "group": "Node", "used_for": "Used for monitoring CPU utilization across individual nodes in the dataplane cluster. Helps identify nodes experiencing high CPU load, enabling quick detection of resource bottlenecks and uneven workload distribution across the infrastructure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_cpu_utilisation:rate1m{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3927, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST files being backed up from TiKV's write column family. Helps diagnose backup performance issues and identify whether large SST files are causing backup slowdowns or bottlenecks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 75, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "Used for monitoring the rate of regions with active write operations on each TiKV instance. Helps identify write hotspots and uneven write distribution across the cluster, which can indicate load balancing issues or hotspot regions requiring attention.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Usage", "group": "Node", "used_for": "Used for monitoring memory utilization ratio across individual nodes in the dataplane cluster. Helps identify nodes experiencing memory pressure or imbalanced memory consumption that could impact workload performance and stability.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "sum(instance:node_memory_utilisation:ratio{cluster=\"$cluster\"}) by (instance)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3924, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU consumption by TiKV backup worker threads and I/O operations during backup tasks. Essential for identifying backup performance bottlenecks, detecting resource contention during backup operations, and ensuring backups don't excessively impact cluster performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"b.*k.*w.*k.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_io\"}[2m])) by (instance)", "tikv_backup_softlimit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "Dataplane Resource Overview", "title": "Deployment", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for each deployment in the specified namespace. Helps verify deployment scaling configurations and identify discrepancies between desired and actual replica counts during capacity planning or troubleshooting deployment issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_deployment_spec_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 33, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total storage size of each RocksDB column family (CF) across TiKV instances. Essential for tracking data distribution, identifying storage growth patterns, and diagnosing disk space issues by showing the breakdown of default, write, lock, and raft column families.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3926, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of backup threads allocated in each TiKV instance's thread pool. Essential for diagnosing backup performance issues and ensuring adequate thread resources are available during backup operations, especially when backups are slow or timing out.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region across TiKV instances. Helps identify uneven write distribution, hotspot regions, and instances experiencing abnormally high or low write activity that may indicate load imbalance or performance issues.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5597, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying bottlenecks when creating cluster snapshots.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4, "dashboard": "Dataplane Resource Overview", "title": "StatefulSet", "group": "Pod", "used_for": "Used for monitoring the desired number of replicas configured for StatefulSets in the selected namespace. Helps verify StatefulSet scaling configurations and identify discrepancies between desired and actual replica counts during deployment or scaling operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3647, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the rate of keys written per second across all regions in TiKV instances. Helps identify write hotspots, uneven data distribution, and diagnose performance issues related to region-level write activity.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763571993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "cloud request", "group": "Backup & Import", "used_for": "Used for monitoring the rate of cloud storage requests (such as S3 or GCS operations) made by TiKV during backup and restore operations. Helps identify cloud API call patterns and diagnose issues with external storage interactions during data import/export tasks.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_cloud_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cloud, req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Dataplane Resource Overview", "title": "Node Memory Request Ratio", "group": "Node", "used_for": "Used for monitoring the ratio of requested memory resources to allocatable memory capacity on each Kubernetes node. Helps identify nodes approaching memory resource limits and potential scheduling constraints before pods fail to deploy due to insufficient memory requests.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "tidb_cluster" ], "promql": [ "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"memory\",unit=\"byte\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3929, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the rate at which TiKV generates SST (Sorted String Table) files during backup operations, measured in bytes per second. Essential for tracking backup performance, identifying throughput bottlenecks, and diagnosing slow backup issues across the cluster or specific instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 57, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region across TiKV instances. Helps identify unbalanced write workloads, hotspot regions, and assess whether data distribution is optimal for write performance.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3646, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the total rate of bytes written to regions across all TiKV instances in the cluster. Helps identify write-heavy workloads, detect abnormal write traffic spikes, and diagnose I/O performance bottlenecks at the region level.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "Dataplane Resource Overview", "title": "Root Disk Usage", "group": "Node", "used_for": "**Used For:**\n\nUsed for monitoring root filesystem disk space utilization across cluster nodes. Essential for preventing disk full conditions that could cause node failures, service disruptions, or data write failures.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster" ], "promql": [ "1 - sum(node_filesystem_avail_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device) / sum(node_filesystem_size_bytes{fstype!=\"\", cluster=\"$cluster\"}) by (instance, device)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 20, "dashboard": "Dataplane Resource Overview", "title": "DaemonSet", "group": "Pod", "used_for": "Used for monitoring the number of pods currently scheduled for each DaemonSet in the selected namespace. Helps verify that DaemonSets are properly deployed across nodes and identify scheduling issues when the actual pod count doesn't match expected node coverage.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_daemonset_status_current_number_scheduled{cluster=\"$cluster\",namespace=\"$namespace\"}) by (daemonset)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 6905, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of saving SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance issues and identifying bottlenecks in the backup data persistence process.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the rate of channel full events in TiKV instances, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, thread pool saturation, or processing bottlenecks that can lead to request timeouts and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5266, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "Backup & Import", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in the default column family during TiKV backup operations. Helps diagnose backup performance issues and identify whether SST file sizes are within optimal ranges for efficient backup processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dataplane Resource Overview", "title": "Pod Status", "group": "Pod", "used_for": "Used for monitoring the current status phase of pods in the specified namespace. Helps quickly identify pod health issues by showing which pods are in Running, Pending, Failed, or other phases, essential for troubleshooting deployment and scheduling problems.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod, phase)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5264, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Errors", "group": "Backup & Import", "used_for": "Used for monitoring the rate of backup operation errors in TiKV instances over 2-minute intervals. Essential for quickly identifying backup failures, diagnosing data protection issues, and ensuring backup reliability in the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1481, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the distribution of Region sizes in TiKV, showing p95, p99, and average values. Essential for identifying Region size imbalances, detecting abnormally large Regions that may cause hotspots or performance degradation, and determining if Region splitting is working properly.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3931, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying slow scan operations that may impact backup completion time.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9560, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft Entry Cache Evicts", "group": "Server", "used_for": "Used for monitoring the rate at which Raft log entries are evicted from TiKV's in-memory cache. High eviction rates indicate memory pressure or insufficient cache size, which can impact Raft replication performance and increase disk I/O for log retrieval.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_raft_entries_evict_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 50, "dashboard": "Dataplane Resource Overview", "title": "Memory Utilization", "group": "Pod", "used_for": "Used for monitoring memory utilization percentage of pods in the dataplane by comparing actual memory usage (working set) against configured resource limits. Helps identify pods approaching memory limits, potential OOM risks, and opportunities for resource allocation optimization.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\"})by(pod)/\nsum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}\n)\n/\nsum by (pod) (\n kube_pod_container_resource_limits{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"POD\"}\n)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 10, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Requests", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring memory resource requests and limits configured for pods in the dataplane namespace. Helps identify pod memory allocation settings and detect potential resource constraint issues when actual usage approaches or exceeds configured limits.", "component": "", "phenomenon": "memory", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"memory\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4936, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the P99 latency of external storage connection initialization in TiKV during backup and restore operations. Critical for diagnosing slow backup/restore performance caused by delays in establishing connections to external storage systems like S3 or GCS.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3720, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of requests received by TiKV servers, showing both average and p99 values by request type. Helps diagnose request batching efficiency and identify whether clients are sending requests individually or in batches, which impacts network overhead and processing performance.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23, "dashboard": "Dataplane Resource Overview", "title": "Pod Memory Usage", "group": "Pod", "used_for": "Used for monitoring the working set memory usage of individual pods in the TiDB cluster. Essential for identifying memory-intensive pods, detecting memory leaks, and ensuring pods stay within resource limits to prevent OOM kills.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3730, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Hibernate Peers", "group": "Server", "used_for": "Used for monitoring the count and state of hibernated Raft peers in TiKV instances. Helps diagnose region management efficiency and identify potential issues with peer hibernation mechanisms that optimize resource usage for idle regions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_hibernated_peer_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3928, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, tracking p95, p99, and average durations. Critical for diagnosing backup performance issues and identifying slow backup ranges that could impact backup completion time or cluster performance during backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7266, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory trace", "group": "Server", "used_for": "Used for tracking detailed memory usage breakdown in TiKV, specifically monitoring memory consumed by Raftstore threads and Raft Engine components. Essential for diagnosing memory pressure issues and identifying which internal TiKV subsystems are consuming excessive memory resources.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}", "raft_engine_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "Dataplane Resource Overview", "title": "CPU Utilization", "group": "Pod", "used_for": "Used for monitoring CPU utilization of pods in the dataplane, showing both actual usage against limits and resource requests against node capacity. Essential for identifying CPU bottlenecks, over-provisioned pods, and node resource pressure that could impact cluster performance.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)/\nsum(kube_pod_container_resource_limit_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\"}[$__rate_interval]\n ) and on (container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)\n/\nsum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\"}\n)", "sum by (node) (kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",container!=\"\",container!=\"POD\"}) / on (node) group_left() kube_node_status_allocatable{cluster=~\"$cluster|$tidb_cluster\",resource=\"cpu\",unit=\"core\"}\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 3410, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Blob GC duration", "group": "Titan - $titan_db", "used_for": "Used for monitoring the garbage collection latency in TiKV's Titan storage engine, which manages large values stored separately from RocksDB. Critical for diagnosing performance issues related to Titan blob file cleanup, showing average, P95, and P99 GC durations to identify when blob garbage collection becomes a bottleneck.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "titan_db" ], "promql": [ "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_average\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile95\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile99\"})", "avg(tikv_engine_blob_gc_micros_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_max\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12971, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Resolve address duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of address resolution operations in TiKV servers. Critical for diagnosing network-related issues, DNS resolution delays, or service discovery problems that could impact inter-node communication and cluster connectivity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_address_resolve_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572692, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the average time tasks spend waiting in TiKV's thread pool queues before execution. Critical for identifying thread pool saturation and scheduling bottlenecks that can cause request processing delays and overall performance degradation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name) / sum(rate(tikv_yatp_pool_schedule_wait_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3930, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "External Storage Create Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of external storage creation operations in TiKV during backup and restore processes. Essential for diagnosing performance issues when establishing connections to external storage systems like S3 or GCS, which can impact backup/restore job startup times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Usage", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of individual pods within the selected namespace. Helps identify pods with high CPU consumption, detect resource bottlenecks, and diagnose performance issues or resource contention at the pod level.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "sum by (cluster,namespace,pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\", container!=\"POD\",container!=\"\"}[$__rate_interval])) " ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checksum Request Duration", "group": "Backup & Import", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze requests in TiKV's coprocessor. Critical for diagnosing performance issues during backup operations, table checksums, and ANALYZE TABLE commands that verify data integrity.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[2m])) by (le,req))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5269, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO Utilization", "group": "Backup & Import", "used_for": "Used for monitoring disk I/O utilization percentage on TiKV nodes during backup and import operations. Critical for identifying disk bottlenecks that can cause latency spikes when performing data-intensive backup or bulk import tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5932, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Errors", "group": "Backup & Import", "used_for": "Used for monitoring the rate of errors occurring during data import operations in TiKV. Essential for quickly identifying import failures or issues when restoring backups or bulk loading data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24, "dashboard": "Dataplane Resource Overview", "title": "Pod CPU Request", "group": "Pod", "used_for": "**Used For:**\n\nUsed for monitoring CPU resource requests and limits configured for each pod in the dataplane namespace. Essential for capacity planning, identifying pods with insufficient resource allocations, and detecting potential CPU throttling issues before they impact performance.", "component": "", "phenomenon": "cpu", "object": "request", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kube_pod_container_resource_requests{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)", "sum(kube_pod_container_resource_limits{cluster=\"$cluster\",namespace=\"$namespace\",resource=\"cpu\"}) by (pod)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 52, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Current Usage", "group": "Volume", "used_for": "Used for monitoring the current storage consumption of PersistentVolumes in the dataplane namespace. Helps identify volumes approaching capacity limits and potential storage exhaustion issues before they impact cluster operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "sum(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim) - sum(kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (persistentvolumeclaim)" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572581, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Thread Pool Schedule Wait Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile wait time before tasks are scheduled in TiKV's thread pools. Critical for diagnosing thread pool saturation and identifying when tasks are queuing excessively before execution, which can indicate resource contention or insufficient thread pool capacity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, name))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolume Size", "group": "Volume", "used_for": "Used for monitoring the requested storage capacity of PersistentVolumeClaims in the specified namespace. Helps track storage allocation, identify capacity planning needs, and detect potential storage exhaustion issues before they impact database operations.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_resource_requests_storage_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572784, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO time per second", "group": "Server", "used_for": "Used for monitoring the rate of disk block read time per second in TiKV's RocksDB storage engine, broken down by request type (storage and coprocessor operations). Essential for diagnosing disk I/O bottlenecks and identifying whether slow queries are caused by storage layer read latency or coprocessor processing delays.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_time\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5925, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU Utilization", "group": "Backup & Import", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import or restore operations. Essential for diagnosing performance bottlenecks during backup restoration, Lightning imports, or BR (Backup & Restore) operations by tracking per-instance CPU consumption and thread activity.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5926, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Thread Count", "group": "Backup & Import", "used_for": "Used for monitoring the number of active SST (Sorted String Table) import threads in TiKV instances during data import or restore operations. Essential for diagnosing import performance issues and verifying that backup/restore jobs are actively processing data across the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "Dataplane Resource Overview", "title": "PersistentVolumeClaim", "group": "Volume", "used_for": "Used for monitoring the inventory and metadata of PersistentVolumeClaims (PVCs) in the specified namespace. Helps administrators track storage claims, verify PVC configurations, and identify storage provisioning issues in the Kubernetes dataplane.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_persistentvolumeclaim_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572785, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Disk IO bytes per second", "group": "Server", "used_for": "Used for monitoring the disk I/O throughput in bytes per second for RocksDB read operations across TiKV instances. Essential for diagnosing disk performance bottlenecks, identifying I/O-intensive workloads, and correlating storage layer read activity with query performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", metric=\"block_read_byte\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Ops", "group": "Backup & Import", "used_for": "Used for monitoring the rate of import RPC operations per second in TiKV, broken down by instance and request type. Essential for tracking data import activity and diagnosing performance issues during backup restoration or bulk data loading operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (instance, request)", "sum(rate(tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request!=\"switch_mode\"}[2m])) by (request)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 98, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message processing in TiKV, broken down by message type. Critical for identifying slow gRPC operations and diagnosing communication bottlenecks between TiDB/TiKV components or client-server interactions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))", "histogram_quantile(0.99, tikv_grpc_msg_duration_seconds_bucket:le_type:sum:rate{type!=\"kv_gc\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 95, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of different gRPC message types processed by TiKV instances. Helps identify which gRPC operations (like Coprocessor requests, Raft messages, or transaction commands) are most active and diagnose communication bottlenecks or unusual request patterns between TiDB/TiKV components.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5929, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Wait Duration", "group": "Backup & Import", "used_for": "Used for monitoring the queue wait time during data import operations in TiKV. Helps diagnose import performance bottlenecks by revealing how long import tasks spend waiting in the queue before being processed.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 107, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiKV, broken down by message type. Critical for identifying communication failures between TiKV and other components (TiDB, PD, or other TiKV nodes), helping diagnose network issues, timeout problems, or service unavailability.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5931, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the 99th percentile latency of import RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets through tools like TiDB Lightning or BR (Backup & Restore), helping identify slow import requests that may impact overall import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))", "histogram_quantile(0.5, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2532, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average gRPC message duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of different gRPC message types in TiKV. Helps identify which gRPC operations (like coprocessor requests, prewrite, commit) are experiencing performance degradation and contributing to overall cluster latency issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 67, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb)", "group": "", "used_for": "Used for monitoring Horizontal Pod Autoscaler (HPA) status for TiDB instances, showing desired, current, and maximum replica counts. Essential for tracking auto-scaling behavior and ensuring TiDB pods scale appropriately based on workload demands.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5930, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, helping identify whether slow imports are caused by write or download operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5928, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the duration of SST (Sorted String Table) file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data or migrating large datasets, as slow rewrite operations can significantly impact overall import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2533, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the batch size of gRPC requests and responses in TiKV, showing both p99 percentile and average values. Helps diagnose gRPC batching efficiency and identify potential network communication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573090, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC resource group QPS", "group": "gRPC", "used_for": "Used for monitoring the queries per second (QPS) for each resource group in TiKV's gRPC layer. Helps identify resource group traffic distribution and diagnose whether specific resource groups are experiencing abnormal request rates or potential resource contention issues.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_resource_group_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "Dataplane Resource Overview", "title": "Avg Mem Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring average memory utilization of TiDB pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether memory-based autoscaling is functioning correctly or if pods are consistently over/under-provisioned.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "1", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"memory\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n container_memory_working_set_bytes{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"} and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"memory\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 5938, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, as high ingestion duration directly impacts import throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "Dataplane Resource Overview", "title": "Docker Images", "group": "Image Versions", "used_for": "Used for monitoring the container images deployed across pods in the dataplane namespace. Helps track image versions, identify version inconsistencies, and verify successful deployments or rollouts across the cluster.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_pod_container_info{cluster=\"$cluster\",namespace=\"$namespace\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 2534, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages sent between TiKV nodes via gRPC. Helps diagnose Raft communication efficiency and identify potential network batching issues that could impact cluster consensus performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 24763573092, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of Raft messages received by each TiKV store instance. Helps diagnose inter-node communication issues, network bottlenecks, and abnormal Raft message traffic patterns that could indicate cluster instability or replication problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_message_recv_by_store{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, store)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5937, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "Backup & Import", "used_for": "Used for monitoring the rate of SST (Sorted String Table) file ingestion during data import operations in TiKV. Essential for tracking import throughput and diagnosing performance bottlenecks when restoring backups or bulk loading data into the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tidb)", "group": "", "used_for": "Used for monitoring TiDB pod CPU utilization in relation to Horizontal Pod Autoscaler (HPA) targets. Helps operators verify if auto-scaling is working correctly by comparing actual CPU usage against configured HPA thresholds, essential for diagnosing scaling behavior and resource allocation issues.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tidbgroup-db-[0-9]+\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\",\"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n) * 100\n", "(\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n /\n sum by (hpa_name) (\n label_replace(\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-[0-9]+-tidb-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tidbgroup-$1\", \"pod\", \"^(db-[^-]+)-tidb-[a-z0-9]+$\"\n )\n )\n) * 100" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 23763572859, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources duration", "group": "gRPC", "used_for": "Used for monitoring the duration of gRPC requests grouped by their source in TiKV. Helps identify which client components (TiDB, PD, or other TiKV nodes) are generating the most gRPC traffic and diagnose source-specific communication bottlenecks or latency issues.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_request_source_duration_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tidb-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiDB worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for verifying autoscaling behavior and identifying when worker pods are scaling up/down in response to workload changes or hitting configured capacity limits.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tidbgroup-db-.*-worker\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-worker-tidb-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 6906, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Read SST Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency of reading SST (Sorted String Table) files during data import operations in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the data ingestion pipeline, particularly when restoring backups or migrating data.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572861, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL Expired", "group": "Backup & Import", "used_for": "Used for monitoring the count of expired TTL (Time-To-Live) entries during backup operations in TiKV. Helps identify data expiration patterns and potential issues with TTL-based data cleanup that could impact backup performance and storage efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_backup_raw_expired_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 79, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's async apply threads, which handle applying Raft log entries to the RocksDB storage engine. Essential for diagnosing write performance bottlenecks and identifying instances where apply threads are saturated, which can cause write stalls and increased latency.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5927, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "Backup & Import", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during data import operations in TiKV. Helps diagnose slow import performance by identifying network bottlenecks or bandwidth limitations when downloading SST files from external sources.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572858, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC request sources QPS", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC requests to TiKV broken down by source (e.g., TiDB, PD, other TiKV nodes). Helps identify which components are generating the most traffic and diagnose abnormal request patterns or unexpected traffic sources.", "component": "TIKV", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_grpc_request_source_counter_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (source)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's Raft store threads, which handle Raft consensus operations and region management. Critical for diagnosing performance bottlenecks when Raft replication becomes slow or when experiencing high write latency due to consensus overhead.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(raftstore|rs)_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 61, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of coprocessor-worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Helps determine if HPA scaling policies are appropriately configured and whether coprocessor workers need scaling adjustments to handle analytical query workloads efficiently.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 1.1", "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"} * 0.9", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-coprocessor-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-coprocessor)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 75, "dashboard": "Dataplane Resource Overview", "title": "Scaling Event Distribution(coprocessor-worker)", "group": "", "used_for": "Used for tracking horizontal pod autoscaler (HPA) scaling events for coprocessor workers in the TiDB dataplane. Helps identify when and how frequently the coprocessor worker pool scales up or down in response to workload changes, essential for understanding autoscaling behavior and capacity adjustments.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace" ], "promql": [ "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} > bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])", "sum_over_time((kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} < bool kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\", horizontalpodautoscaler=~\".*-coprocessor\"} offset 15s)[$__range:15s])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 70, "dashboard": "Dataplane Resource Overview", "title": "HPA info(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of coprocessor worker pods, showing desired, current, and maximum replica counts. Essential for tracking autoscaling behavior and ensuring coprocessor workers scale appropriately to handle workload demands without hitting configured limits.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\".*-coprocessor\"}", "sum(kube_pod_status_ready{cluster=\"$cluster\", namespace=\"$namespace\", condition=\"true\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{phase=\"Running\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})", "sum(kube_pod_status_phase{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"db-.*-coprocessor-tikv-worker-.*\"})" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 13115, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store writer CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's store writer threads, which handle Raft log writes and apply operations to RocksDB. Essential for diagnosing write bottlenecks and identifying instances where store writer threads are saturated, potentially causing write stalls or increased latency.", "component": "TIKV", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_write.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12309, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write keys", "group": "Backup & Import", "used_for": "Used for monitoring the number of keys written during local import operations in TiKV over 2-minute intervals. Helps track import progress and diagnose slow data ingestion issues when using TiDB Lightning or BR (Backup & Restore) local backend mode.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_local_write_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 63, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tidb_worker_hint(tidb-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiDB worker pods, comparing target scaling thresholds against current metric values. Helps operators understand when and why TiDB workers are being scaled up or down based on the custom `dxf_hscale_tidb_worker_hint` metric in Kubernetes environments.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"} *1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tidb_worker_hint\",\n horizontalpodautoscaler=~\"tidbgroup-db-.*.-worker\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 73, "dashboard": "Dataplane Resource Overview", "title": "Scale Out summay[WIP](coprocessor-worker)", "group": "", "used_for": "Used for monitoring the autoscaling behavior of TiDB coprocessor worker pods in Kubernetes environments. Tracks CPU utilization thresholds, scaling events, and desired replica changes to help diagnose whether horizontal pod autoscaling is responding appropriately to workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__range", "cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", " kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1", " delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n max_over_time(\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[30s]\n )\n <=\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) > 0\n )\n )[$__range:15s]\n))", "sum(count_over_time(\n (\n (\n kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }\n >\n kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\",\n metric_target_type=\"utilization\",\n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n } * 1.1\n )\n and on(horizontalpodautoscaler)\n (\n delta(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\"db-.*-coprocessor-worker\"\n }[15s]\n ) <= 0\n )\n )[$__range:15s]\n))" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 12310, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Local Write bytes", "group": "Backup & Import", "used_for": "Used for monitoring the rate of data bytes written during TiDB Lightning's local import process in TiKV. Helps diagnose import performance issues and track data ingestion throughput when using the local backend import mode.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_local_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4464, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiKV's encryption-at-rest system. Helps verify encryption key management health and diagnose issues with data key generation, rotation, or storage capacity.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC threads in TiKV instances, which handle client requests and inter-node communication. Essential for identifying CPU bottlenecks in request processing and diagnosing performance issues when gRPC threads become saturated under high load.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 77, "dashboard": "Dataplane Resource Overview", "title": "Scaling Evnt Detials(coprocessor-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaling behavior of coprocessor workers, tracking current replicas, desired replicas, and the scaling gap. Essential for diagnosing autoscaling responsiveness issues and determining whether coprocessor resources are scaling appropriately to meet workload demands.", "component": "", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "__interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "last_over_time(kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])", "last_over_time(kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n}[$__interval:$__interval])\n", "(\n kube_horizontalpodautoscaler_status_desired_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n -\n kube_horizontalpodautoscaler_status_current_replicas{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\"\n }\n) != 0", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval])\n", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset -15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"\n}[$__interval:$__interval] offset 15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval])", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset -15s)", "last_over_time(\n histogram_quantile(0.99, sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n)) by (le))[$__interval:$__interval] offset 15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval])\n", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset -15s)", "last_over_time(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count{\n tidb_cluster_id=\"$tidb_cluster\"\n }[$__interval]\n))[$__interval:$__interval] offset 15s)", "last_over_time(kube_horizontalpodautoscaler_status_target_metric{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n horizontalpodautoscaler=~\".*-coprocessor\",\n metric_name=\"cpu\",\n metric_target_type=\"utilization\"}[$__interval:$__interval])\n/\nlast_over_time(kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\".*-coprocessor\"}[$__interval:$__interval])" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 79, "dashboard": "Dataplane Resource Overview", "title": "HPA Evaluation Result(coprocessor-worker)", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the Horizontal Pod Autoscaler (HPA) evaluation results for coprocessor worker pods in the dataplane. Helps operators understand autoscaling decisions and verify whether the system is correctly scaling coprocessor resources based on workload demands.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4556, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the disk space consumed by TiKV's encryption metadata files. Helps identify storage overhead from encryption key management and detect potential issues with metadata file growth that could impact disk capacity.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4557, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypt/decrypt data nanos", "group": "Encryption", "used_for": "Used for monitoring the time spent on encrypting and decrypting data in TiKV's RocksDB storage engine, broken down by request type. Essential for diagnosing encryption-at-rest performance overhead and identifying whether encryption/decryption operations are causing I/O bottlenecks or latency issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"encrypt_data_nanos\"}[2m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"decrypt_data_nanos\"}[2m])) by (req)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4554, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiKV's storage engine. Helps verify encryption-at-rest is working correctly and track the volume of encrypted data, useful when diagnosing encryption configuration issues or validating security compliance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether TiKV's encryption-at-rest feature has been successfully initialized. Essential for confirming encryption setup status during cluster deployment or when troubleshooting encryption-related configuration issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 69, "dashboard": "Dataplane Resource Overview", "title": "HPA info(tikv-worker)", "group": "", "used_for": "Used for monitoring the Horizontal Pod Autoscaler (HPA) status of TiKV worker pods, showing desired vs. current replica counts and maximum scaling limits. Essential for tracking auto-scaling behavior and identifying when TiKV workers are scaling up/down in response to load changes or hitting configured capacity limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace" ], "promql": [ "kube_horizontalpodautoscaler_status_desired_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_status_current_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_max_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "kube_horizontalpodautoscaler_spec_min_replicas{cluster=\"$cluster\", namespace=\"$namespace\",horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 14507, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Average Flush Size ", "group": "Backup Log", "used_for": "Used for monitoring the average size of backup log files flushed to storage in TiKV's log backup feature. Helps identify backup efficiency issues and diagnose abnormal flush patterns that could indicate storage bottlenecks or configuration problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]) / on(instance) increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14361, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Endpoint Status", "group": "Backup Log", "used_for": "Used for monitoring the operational status of TiKV's log backup endpoint, indicating whether log backup is enabled on each instance. Essential for verifying backup configuration and quickly identifying instances where log backup may be disabled or misconfigured.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_enabled{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 16, "dashboard": "Dedicated Dataplane Node Info", "title": "Node Info", "group": "", "used_for": "Used for displaying basic node information in Kubernetes clusters, including node names, kernel versions, and OS details. Helps operators quickly identify node characteristics and verify cluster configuration when troubleshooting infrastructure-level issues.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "instance", "k8s_cluster" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node,k8s_cluster_info) group_right kube_node_info" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 69, "dashboard": "Dedicated Dataplane Node Info", "title": "Notice", "group": "", "used_for": "**Used For:**\n\nUsed for displaying important notices, warnings, or configuration information about dedicated dataplane nodes. Provides contextual guidance or alerts that operators should be aware of when monitoring the dataplane infrastructure.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 62, "dashboard": "Dataplane Resource Overview", "title": "dxf_hscale_tikv_worker_hint(tikv-worker)", "group": "", "used_for": "Used for monitoring the horizontal pod autoscaler (HPA) metrics for TiKV worker pods, comparing target thresholds against current metric values. Helps operators understand autoscaling behavior and diagnose whether TiKV workers are scaling appropriately based on the custom `dxf_hscale_tikv_worker_hint_v2` metric.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_name=\"dxf_hscale_tikv_worker_hint_v2\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 60, "dashboard": "Dataplane Resource Overview", "title": "Avg CPU Util By HPA(tikv-worker)", "group": "", "used_for": "Used for monitoring the average CPU utilization of TiKV worker pods managed by Horizontal Pod Autoscaler (HPA), comparing actual usage against configured target thresholds. Essential for validating HPA scaling decisions and identifying whether TiKV pods need autoscaling adjustments to handle workload demands efficiently.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "1", "__rate_interval", "cluster", "namespace", "tidb_cluster" ], "promql": [ "kube_horizontalpodautoscaler_spec_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\", \n horizontalpodautoscaler=~\"tikvworker-.*-default\"} * 1.1", "kube_horizontalpodautoscaler_status_target_metric{\n cluster=~\"$cluster|$tidb_cluster\",\n namespace=\"$namespace\", \n metric_target_type=\"utilization\", \n metric_name=\"cpu\",\n horizontalpodautoscaler=~\"tikvworker-.*-default\"}", "avg by (hpa_name) (\n label_replace(\n sum by(pod) (\n rate(\n container_cpu_usage_seconds_total{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",image!=\"\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\"}[$__rate_interval]\n ) and on(container,pod) kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n )\n /\n sum by(pod) (\n kube_pod_container_resource_requests{cluster=~\"$cluster|$tidb_cluster\",namespace=\"$namespace\",resource=\"cpu\",container!=\"\",container!=\"POD\",pod=~\"db-.*-default-tikv-worker-.*\",node!=\"\"}\n ),\n \"hpa_name\", \"tikvworker-$1\", \"pod\",\"^(db-[^-]+-default)-tikv-worker-[a-z0-9]+$\"\n )\n) * 100\n" ], "dashboard_file": "clinic-resource-overview.json" }, { "panel_id": 4559, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the p95 and average latency of encryption metadata read/write operations in TiKV. Critical for diagnosing performance overhead introduced by encryption-at-rest, helping identify if encryption metadata I/O is causing slowdowns in encrypted TiKV clusters.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14, "dashboard": "Dedicated Dataplane Node Info", "title": "Node labels", "group": "", "used_for": "Used for displaying Kubernetes node labels and metadata for dedicated dataplane nodes. Helps operators verify node configurations, identify node roles, and troubleshoot node-specific issues by correlating system metrics with Kubernetes label assignments.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "0", "2", "instance", "k8s_cluster", "namespace", "pod" ], "promql": [ "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\", \"$0\", \"instance\", \".*\") * on(node, k8s_cluster_info) group_right sum(kube_node_labels) by (node, k8s_cluster_info, label_zone, label_dedicated, label_failure_domain_beta_kubernetes_io_zone, label_beta_kubernetes_io_instance_type)", "(\n label_replace(\n node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"},\n \"node\", \"$0\", \"instance\", \".*\"\n )\n * on(node, k8s_cluster_info) group_right\n sum(kube_node_labels)\n by (node, k8s_cluster_info,\n label_zone, label_dedicated,\n label_failure_domain_beta_kubernetes_io_zone,\n label_beta_kubernetes_io_instance_type)\n)\n* on(node) group_left(pod, component)\n label_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\"$pod\"},\n \"component\", \"$2\", \"pod\",\n \"(^|.*-)(tidb|tikv|tiflash|pd|tso|ticdc)(-[0-9a-z]+)*$\"\n )" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14363, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flushed Files (Last 30m) Per Host", "group": "Backup Log", "used_for": "Used for monitoring the number of backup log files flushed to storage by each TiKV instance in the last 30 minutes. Helps track backup log activity patterns and identify instances with abnormal flush behavior or potential backup performance issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14508, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Times (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the number of backup log flush operations completed in the last 30 minutes on TiKV instances. Helps track backup log activity frequency and identify potential issues with log backup file persistence or performance degradation in the backup process.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(sum(increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 81, "dashboard": "Dedicated Dataplane Node Info", "title": "Component Node Number Trends", "group": "", "used_for": "Used for tracking the number of active nodes for each TiDB component (TiDB, TiKV, TiFlash) over time in a dedicated dataplane deployment. Helps monitor cluster scaling events, detect unexpected node failures, and verify that the expected number of component instances are running.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "3", "namespace" ], "promql": [ "count(\n label_replace(\n kube_pod_info{namespace=\"$namespace\", pod=~\"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"},\n \"component\", \n \"$1$3\", \n \"pod\", \n \"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"\n )\n) by (component)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14907, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Status", "group": "Backup Log", "used_for": "Used for monitoring the status of TiKV's log backup tasks across instances. Essential for quickly identifying backup task failures or abnormal states that could impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "min(tikv_log_backup_task_status{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received", "group": "", "used_for": "Used for monitoring the network ingress traffic rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual traffic spikes, or connectivity issues that may impact data transfer performance between TiDB components.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15361, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Advancer Owner", "group": "Backup Log", "used_for": "Used for identifying which TiKV instance is currently serving as the log backup advancer owner. Essential for troubleshooting log backup coordination issues and verifying proper leader election in the backup subsystem.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "tidb_log_backup_advancer_owner > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted", "group": "", "used_for": "Used for monitoring the network transmission rate (bytes per second) on dedicated dataplane nodes. Helps identify network bandwidth saturation, unusual outbound traffic patterns, or potential network bottlenecks affecting data replication and client communication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 47, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Transmitted Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets during network transmission on dataplane nodes. Essential for identifying network congestion, buffer overflow issues, or hardware problems that prevent packets from being sent successfully.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_transmit_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14362, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Total Flushed Size (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total amount of backup log data flushed to storage in the last 30 minutes across TiKV instances. Helps track backup progress, identify backup throughput issues, and diagnose problems with log backup operations when data volume appears abnormally low or high.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 54, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_in_allowance_exceeded", "group": "", "used_for": "Used for monitoring network bandwidth ingress throttling events on dedicated dataplane nodes. Helps identify when incoming network traffic exceeds allocated bandwidth allowances, which can cause packet drops and network performance degradation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_in_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 56, "dashboard": "Dedicated Dataplane Node Info", "title": "bw_out_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring network bandwidth throttling events when outbound traffic exceeds allocated limits on dedicated dataplane nodes. Critical for diagnosing network performance degradation caused by bandwidth quota enforcement and identifying nodes that may need increased network capacity allocation.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_bw_out_allowance_exceeded{cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13262, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CPU Usage", "group": "Backup Log", "used_for": "Used for monitoring CPU usage of TiKV's backup log threads, including backup stream and log backup scan operations. Essential for identifying resource bottlenecks during continuous backup processes and ensuring backup operations don't impact cluster performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 46, "dashboard": "Dedicated Dataplane Node Info", "title": "Network Received Drop", "group": "", "used_for": "Used for monitoring the rate of dropped packets on network interfaces during reception on dedicated dataplane nodes. Critical for identifying network congestion, buffer overflow issues, or hardware problems that could impact cluster communication and data transfer reliability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_network_receive_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14911, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Files (Last 30m)", "group": "Backup Log", "used_for": "Used for monitoring the total number of log backup files flushed to external storage in the last 30 minutes across TiKV instances. Helps diagnose backup progress issues, identify instances with abnormal flush rates, and verify that log backup is actively writing data during troubleshooting.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "round(sum(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m])))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12843, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Handle Event Rate", "group": "Backup Log", "used_for": "Used for monitoring the rate at which TiKV processes key-value batches during log backup operations. Helps identify backup throughput issues and diagnose performance bottlenecks in the continuous data change capture process.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "rate(tikv_log_backup_handle_kv_batch_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 99, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Table Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the connection tracking table utilization ratio on dataplane nodes. Critical for identifying when the conntrack table approaches capacity limits, which can cause connection drops and network issues in high-traffic Kubernetes environments.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "node_nf_conntrack_entries{cluster=\"$k8s_cluster\",instance=~\"$instance\"} / node_nf_conntrack_entries_limit{cluster=\"$k8s_cluster\",instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13100, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Memory Of Events", "group": "Backup Log", "used_for": "Used for monitoring the heap memory consumption of TiKV's backup log events. Helps identify memory pressure issues during log backup operations and diagnose potential memory leaks or excessive memory usage in the backup subsystem.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_heap_memory{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 57, "dashboard": "Dedicated Dataplane Node Info", "title": "conntrack_allowance_exceeded", "group": "", "used_for": "Used for monitoring connection tracking table overflow events on dedicated dataplane nodes. Helps diagnose network connection issues caused by exceeding the conntrack table limits, which can lead to dropped connections and network instability.", "component": "", "phenomenon": "connection", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_conntrack_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 98, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Drops", "group": "", "used_for": "Used for monitoring the rate of dropped connection tracking entries in the Linux kernel's conntrack subsystem on dataplane nodes. Critical for diagnosing connection loss issues when the conntrack table becomes full, indicating potential network throughput bottlenecks or the need to tune conntrack table size limits.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14135, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan Generate Event Throughput", "group": "Backup Log", "used_for": "Used for monitoring the throughput of initial scan event generation during TiKV log backup operations. Helps diagnose backup performance issues and track the rate at which historical data is being scanned and converted into backup events during the initial backup phase.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(tikv_log_backup_incremental_scan_bytes_sum{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14908, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Fatal Errors", "group": "Backup Log", "used_for": "Used for monitoring fatal errors in TiKV's backup log functionality. Critical for detecting severe backup failures that require immediate attention, helping diagnose issues preventing successful log backup operations.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__interval", "instance" ], "promql": [ "increase(tikv_log_backup_fatal_errors{instance=~\"$instance\"}[$__interval])", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 55, "dashboard": "Dedicated Dataplane Node Info", "title": "linklocal_allowance_exceeded", "group": "", "used_for": "Used for monitoring the rate of link-local address allowance exceeded events on network interfaces in the dataplane node. Helps identify network configuration issues or excessive link-local traffic that may indicate connectivity problems or misconfigurations in the Kubernetes cluster networking layer.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_linklocal_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14630, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Observed Region Count", "group": "Backup Log", "used_for": "Used for monitoring the number of regions being observed by TiKV's backup log feature across instances. Helps diagnose backup coverage issues and verify that all regions are properly tracked during log backup operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance" ], "promql": [ "tikv_log_backup_observed_region{instance=~\"$instance\"}", "sum(tikv_log_backup_observed_region{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 58, "dashboard": "Dedicated Dataplane Node Info", "title": "PPS_allowance_exceeded", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring packet-per-second (PPS) rate limiting violations on network interfaces. Helps identify when network traffic exceeds configured PPS allowances, which can indicate traffic bursts, DDoS attempts, or misconfigured rate limits that may impact cluster network performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "increase(node_ethtool_pps_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14078, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush Duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to flush backup log files to external storage in TiKV. Critical for diagnosing backup performance issues and identifying bottlenecks in the log backup pipeline's file-saving stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_flush_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_files\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "**Used For:**\n\nUsed for monitoring the current memory utilization on dedicated dataplane nodes. Essential for identifying memory pressure, potential OOM risks, and capacity planning for node resources.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(\n (\n (\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n )\n / node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n ) * 100)\n" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14910, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Checkpoint TS of Tasks", "group": "Backup Log", "used_for": "**Used For:**\n\nUsed for monitoring the checkpoint timestamp progress of TiDB log backup tasks in TiKV. Helps verify backup tasks are advancing properly and identify potential backup lag or stalled tasks by comparing checkpoint positions against current time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "max(tidb_log_backup_last_checkpoint{instance=~\"$instance\"} / 262144 > 0) by (task)", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14136, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial scanning duration", "group": "Backup Log", "used_for": "Used for monitoring the time taken to complete initial log scanning during TiKV backup operations. Helps diagnose slow backup startup issues and identify performance bottlenecks in the initial data scanning phase of log backup.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_initial_scan_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12840, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Wait for Lock Duration", "group": "Backup Log", "used_for": "Used for monitoring the time TiKV spends waiting to acquire router locks during log backup operations. Critical for diagnosing backup performance bottlenecks caused by lock contention in the backup log subsystem.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"get_router_lock\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 100, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Insert Failures", "group": "", "used_for": "Used for monitoring the rate of connection tracking table insertion failures on dataplane nodes. Critical for diagnosing network connection issues caused by conntrack table exhaustion, which can lead to dropped connections and increased latency when the system cannot track new network flows.", "component": "", "phenomenon": "latency", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_insert_failed{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13101, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Errors", "group": "Backup Log", "used_for": "Used for monitoring error occurrences in TiKV's log backup process. Essential for identifying backup failures, detecting reliability issues in the backup pipeline, and troubleshooting when log backup operations are not completing successfully.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "__interval", "instance" ], "promql": [ "increase(tikv_log_backup_errors{instance=~\"$instance\"}[$__interval])", "tikv_log_backup_errors{instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the latency distribution of writing backup log data to temporary files in TiKV. Critical for diagnosing performance bottlenecks in the log backup process, particularly when backup operations are slow or experiencing delays due to disk I/O issues.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=\"write_to_tempfile\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13934, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Convert Raft Event duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent converting Raft events to stream events during TiKV log backup operations. Critical for diagnosing backup performance bottlenecks and identifying delays in the event conversion pipeline that could impact backup throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"to_stream_event\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 101, "dashboard": "Dedicated Dataplane Node Info", "title": "Conntrack Early Drops", "group": "", "used_for": "Used for monitoring the rate of connection tracking table entries dropped due to table overflow on dataplane nodes. Critical for diagnosing network connection limits and potential packet loss when the conntrack table reaches capacity under high connection loads.", "component": "", "phenomenon": "throughput", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_nf_conntrack_stat_early_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14774, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Abnormal Checkpoint TS Lag", "group": "Backup Log", "used_for": "Used for monitoring the time lag between the current time and the last backup checkpoint timestamp in TiDB log backup tasks. Critical for detecting backup delays or stalls that could impact point-in-time recovery capabilities and data protection SLAs.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "time() * 1000 - max(tidb_log_backup_last_checkpoint / 262144 > 0) by (task)", "time() * 1000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 45, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk I/O Util", "group": "Disk", "used_for": "Used for monitoring disk I/O utilization percentage on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting saturated storage devices, and diagnosing performance degradation caused by excessive I/O operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 4, "dashboard": "Dedicated Dataplane Node Info", "title": "CPU Utilization", "group": "CPU", "used_for": "Used for monitoring the overall CPU utilization percentage on dedicated dataplane nodes. Essential for identifying CPU bottlenecks, resource saturation, and determining if nodes need scaling or workload rebalancing.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m])) * 100)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 3, "dashboard": "Dedicated Dataplane Node Info", "title": "Usage Per Core", "group": "CPU", "used_for": "Used for monitoring CPU utilization per individual core on dedicated dataplane nodes. Helps identify CPU core imbalances, hotspots, and whether workloads are properly distributed across all available cores.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "sum by (cpu) (irate(node_cpu_seconds_total{ job=\"node-exporter\", mode!=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m]))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13551, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "System Write Call Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration distribution of system write calls during TiKV log backup operations. Essential for diagnosing I/O performance bottlenecks and identifying when disk write latency impacts backup log processing efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=\"syscall_write\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12841, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Save to Temp File Duration", "group": "Backup Log", "used_for": "Used for monitoring the time spent saving backup log data to temporary files in TiKV. Critical for diagnosing performance bottlenecks in the log backup pipeline, particularly when backup operations are slow or experiencing delays in the initial file writing stage.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"save_to_temp_file\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "Dedicated Dataplane Node Info", "title": "Memory Usage", "group": "Memory", "used_for": "Used for monitoring memory consumption on dedicated dataplane nodes, breaking down total used memory, buffer cache, and page cache. Essential for identifying memory pressure, detecting potential OOM risks, and understanding how the system allocates memory between applications and kernel caches.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "node", "port" ], "promql": [ "max(\n node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n)\n", "max(node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$node:$port\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 2, "dashboard": "Dedicated Dataplane Node Info", "title": "System load", "group": "CPU", "used_for": "Used for monitoring the 1-minute, 5-minute, and 15-minute system load averages on dedicated dataplane nodes. Essential for identifying CPU resource saturation and determining if the node is overloaded or experiencing sustained high workload pressure.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "max(node_load1{job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load5{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})", "max(node_load15{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15059, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Command Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of key-value batch sizes processed during TiKV log backup operations. Helps identify batch processing efficiency and diagnose potential performance issues when backup commands are handling unusually large or small batches of data.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__interval", "instance", "tidb_cluster" ], "promql": [ "sum(increase(tikv_log_backup_handle_kv_batch_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 22, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average wait time per disk I/O operation (both reads and writes) on dedicated dataplane nodes. Essential for identifying disk performance bottlenecks and diagnosing storage-related latency issues that may impact TiDB cluster performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14912, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile latency of internal message handling in TiKV's backup log subsystem. Critical for diagnosing performance bottlenecks in the log backup actor system, helping identify which message types are experiencing processing delays that could impact backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance" ], "promql": [ "sum(histogram_quantile(0.99, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14914, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Type", "group": "Backup Log", "used_for": "Used for monitoring the rate of different internal message types processed by TiKV's log backup actor. Helps diagnose log backup performance issues by identifying which message types are most frequently processed or experiencing bottlenecks during backup operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_interal_actor_acting_duration_sec_count{instance=~\"$instance\"}[$__rate_interval])) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14913, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Internal Message Handling Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile latency of internal message handling in TiKV's backup log subsystem. Essential for diagnosing performance bottlenecks in the log backup actor system, helping identify which message types are experiencing processing delays that could impact backup operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(histogram_quantile(0.9, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Wait Latency", "group": "Disk", "used_for": "Used for monitoring the average time spent waiting for disk write operations to complete on dataplane nodes. Essential for identifying disk I/O bottlenecks and diagnosing storage performance degradation that could impact TiKV write throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 91, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk iops", "group": "Disk", "used_for": "Used for monitoring write I/O operations per second on disk devices for dedicated dataplane nodes. Helps identify disk write throughput bottlenecks and diagnose storage performance issues when experiencing slow writes or high I/O wait times.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 24, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk iops", "group": "Disk", "used_for": "Used for monitoring disk I/O operations per second (IOPS) on dedicated dataplane nodes, tracking both read and write operations. Essential for identifying disk bottlenecks, evaluating storage performance, and diagnosing I/O-related slowdowns in the cluster.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14270, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Operation ", "group": "Backup Log", "used_for": "Used for monitoring the rate of RocksDB operations during log backup's initial scan phase in TiKV. Helps diagnose performance bottlenecks and operation patterns when backup starts scanning existing data, excluding read byte metrics to focus on operation counts across different column families.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op!~\"read_bytes\"}[$__rate_interval])) BY (op, cf) > 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15544, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Checkpoint Batch Size", "group": "Backup Log", "used_for": "Used for monitoring the distribution of batch sizes when TiKV processes checkpoint requests during log backup operations. Helps diagnose backup efficiency issues by revealing whether checkpoint batching is optimal or if batch sizes are too small/large, which could impact backup performance and resource utilization.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_advancer_batch_size_bucket{type=\"checkpoint\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 14915, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scanning Trigger Reason", "group": "Backup Log", "used_for": "Used for tracking the reasons that trigger initial scanning operations in TiKV's backup log feature. Helps diagnose why backup scans are initiated (e.g., new region, leader transfer, or configuration changes) and identify potential issues causing excessive or unexpected scan triggers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(increase(tikv_log_backup_initial_scan_reason{instance=~\"$instance\"}[$__rate_interval])) by (reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 32, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of individual disk read and write operations per second on dataplane nodes. Helps identify I/O patterns and detect inefficient small I/O operations that could indicate fragmentation or suboptimal workload behavior.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 26, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring disk I/O bandwidth (read and write rates) on dedicated dataplane nodes. Essential for identifying disk bottlenecks, detecting abnormal I/O patterns, and diagnosing performance issues related to storage throughput in Kubernetes clusters.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 93, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk Bandwidth", "group": "Disk", "used_for": "Used for monitoring the write bandwidth (bytes per second) to physical disks on dedicated dataplane nodes. Essential for identifying disk I/O bottlenecks, detecting write-heavy workloads, and diagnosing storage performance issues that may impact TiKV write operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 92, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Disk IO Avg Size / second", "group": "Disk", "used_for": "Used for monitoring the average size of disk I/O operations (both read and write) per second on dataplane nodes. Helps identify I/O patterns and diagnose whether workloads are performing small random I/O or large sequential I/O, which is critical for understanding storage performance characteristics and potential bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])", "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 14271, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Initial Scan RocksDB Throughput ", "group": "Backup Log", "used_for": "Used for monitoring the data read throughput from RocksDB during the initial scan phase of TiKV log backup operations. Essential for tracking backup performance and diagnosing slow initial backup speeds or bottlenecks in reading data from different column families.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op=~\"read_bytes\"}[$__rate_interval])) BY (op, cf)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 95, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read and write requests per second on disk devices in dedicated dataplane nodes. Helps identify disk I/O optimization efficiency and potential bottlenecks when the OS merges adjacent I/O requests to improve throughput.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763572666, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Failure Reason", "group": "Backup Log", "used_for": "Used for tracking the distribution of non-retryable failure reasons when TiKV attempts to checkpoint regions during log backup operations. Essential for diagnosing why backup log tasks are failing and identifying systematic issues preventing successful region checkpointing.", "component": "TIKV", "phenomenon": "error_rate", "object": "region", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_region_request_failure{reason!=\"retryable-scan-region\"}[$__interval])) by (reason)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15716, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration", "group": "Backup Log", "used_for": "Used for monitoring the duration of backup log advancer tick operations in TiKV. Helps diagnose performance issues with the log backup checkpoint advancement process, which is critical for ensuring timely and consistent point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__interval", "tidb_cluster" ], "promql": [ "sum(increase(tidb_log_backup_advancer_tick_duration_sec_bucket{tidb_cluster_id=\"$tidb_cluster\", step=~\"tick\"}[$__interval])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 30, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Disk (wrqm & rrqm) / second", "group": "Disk", "used_for": "Used for monitoring the rate of merged read (rrqm) and write (wrqm) requests per second on disk devices in the dataplane node. Helps assess I/O efficiency and identify whether the kernel is successfully merging adjacent I/O requests to reduce disk operations and improve performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "rate(node_disk_reads_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])", "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 15359, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P99)", "group": "Backup Log", "used_for": "Used for monitoring the 99th percentile duration of backup log advancement ticks in TiKV. Critical for diagnosing performance issues in the log backup process, particularly when backup operations are slow or experiencing delays in advancing the backup checkpoint.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [], "promql": [ "sum(histogram_quantile(0.99, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 48, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking the percentage of used storage across different devices. Essential for preventing disk full scenarios and identifying nodes approaching storage capacity limits that could impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "((max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) /\n max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) * 100" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763572665, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request Result", "group": "Backup Log", "used_for": "Used for monitoring the results of log backup region requests in TiKV, categorized by success or failure status. Helps diagnose log backup reliability issues and identify request failures that could impact point-in-time recovery capabilities.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "__interval" ], "promql": [ "sum(increase(tidb_log_backup_region_request[$__interval])) by (result)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15176, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Region Checkpoint Key Putting", "group": "Backup Log", "used_for": "Used for monitoring the rate of region checkpoint metadata key operations during TiKV log backup. Helps diagnose backup performance issues and identify bottlenecks in checkpoint metadata persistence, which is critical for tracking backup progress and ensuring data consistency during point-in-time recovery operations.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "sum(rate(tikv_log_backup_metadata_key_operation{instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 15360, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Tick Duration (P90)", "group": "Backup Log", "used_for": "Used for monitoring the 90th percentile duration of backup log advancement ticks in TiKV. Helps diagnose performance issues in the log backup process by identifying when backup operations are taking longer than expected, which could indicate storage bottlenecks or resource contention affecting backup efficiency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "sum(histogram_quantile(0.9, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 49, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Space Used", "group": "Disk", "used_for": "Used for monitoring disk space utilization on dedicated dataplane nodes by tracking used versus available storage across filesystem devices. Essential for preventing disk full conditions and capacity planning, helping identify nodes approaching storage limits before they impact cluster operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster" ], "promql": [ "(max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 97, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiDB persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent file creation even when disk space is available, helping diagnose storage-related failures in containerized TiDB deployments.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 23763572733, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Get Region Operation Count", "group": "Backup Log", "used_for": "Used for monitoring the frequency of region retrieval operations during TiDB log backup processes. Helps diagnose backup performance issues by tracking how often the backup advancer queries region information, which can indicate region scanning bottlenecks or excessive region fragmentation impacting backup efficiency.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(tidb_log_backup_advancer_tick_duration_sec_count{step=\"get-regions-in-range\"}[$__rate_interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup CPU Utilization", "group": "", "used_for": "Used for monitoring CPU utilization of TiKV backup worker threads during backup operations. Essential for identifying backup performance bottlenecks, detecting resource contention during backups, and ensuring backup tasks don't overload the cluster.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 23763572734, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Try Advance Trigger Time", "group": "Backup Log", "used_for": "Used for monitoring the frequency of log backup advancement attempts in TiKV. Helps diagnose backup progress issues and identify when the log backup advancer is actively trying to move the checkpoint forward, which is essential for troubleshooting backup lag or stalling problems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__interval" ], "promql": [ "increase(tidb_log_backup_advancer_tick_duration_sec_count{step=\"try-advance\"}[$__interval])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 51, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC inode usage", "group": "Disk", "used_for": "Used for monitoring inode usage percentage on TiKV and TiFlash persistent volume claims in Kubernetes environments. Critical for detecting inode exhaustion issues that can prevent new file creation even when disk space is available, particularly important for storage engines that create many small files.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 52, "dashboard": "Dedicated Dataplane Node Info", "title": "TiKV/TiFlash PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space utilization of TiKV and TiFlash persistent volume claims in Kubernetes environments. Essential for tracking storage consumption and preventing disk full issues that could cause data write failures or service disruptions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 10, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Errors", "group": "", "used_for": "Used for monitoring the rate of backup operation errors in TiKV instances over 2-minute intervals. Essential for quickly detecting backup failures and diagnosing data protection issues that could impact disaster recovery capabilities.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_backup_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 38, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Default CF SST Size", "group": "", "used_for": "Used for monitoring the size distribution of SST (Sorted String Table) files in TiKV's default column family during backup operations. Helps diagnose backup performance issues and identify abnormally large SST files that could impact backup efficiency or storage requirements.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 96, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB PVC disk usage", "group": "Disk", "used_for": "Used for monitoring disk space consumption of TiDB Persistent Volume Claims (PVCs) in Kubernetes environments. Essential for tracking storage utilization and preventing disk full issues that could impact TiDB server operations and data persistence.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "namespace" ], "promql": [ "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 62, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance", "used_for": "Used for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits on dedicated dataplane nodes. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to I/O throttling and degraded cluster performance.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 37, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Write CF SST Size", "group": "", "used_for": "Used for monitoring the size distribution of SST files being backed up from TiKV's write column family. Helps identify backup I/O patterns and diagnose issues with backup performance or unusually large SST files during backup operations.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 35, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Thread Count", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the number of backup threads allocated in each TiKV instance's backup thread pool. Essential for diagnosing backup performance issues and ensuring adequate thread resources are available during backup operations, especially when backups are slow or experiencing I/O bottlenecks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 64, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring write input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Essential for identifying storage bottlenecks, evaluating whether workloads are approaching EBS IOPS limits, and diagnosing write-heavy performance issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 39, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Save SST Duration", "group": "", "used_for": "Used for monitoring the latency of saving SST (Sorted String Table) files during TiKV backup operations. Critical for diagnosing backup performance bottlenecks and identifying slow file write operations that could impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 8, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IOPS", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read input/output operations per second (IOPS) on EBS volumes attached to dedicated dataplane nodes. Helps identify read-heavy workloads, diagnose storage performance bottlenecks, and verify if EBS volume IOPS limits are being reached during high read activity.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_ops_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 40, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Scan SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file scanning operations during TiKV backup processes. Critical for diagnosing backup performance issues and identifying slow scan operations that may impact backup completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 66, "dashboard": "Dedicated Dataplane Node Info", "title": "Write IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum write I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume write operations are experiencing delays that could impact TiKV write performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 75, "dashboard": "Dedicated Dataplane Node Info", "title": "TiDB Cluster Pods", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for displaying all TiDB cluster pods running in the specified Kubernetes namespace. Helps operators verify pod deployment status and identify which pods are active in the cluster for correlation with EBS volume performance metrics.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "namespace" ], "promql": [ "kube_pod_info{namespace=\"$namespace\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 44, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup Snapshot Duration", "group": "", "used_for": "Used for monitoring the duration of backup snapshot operations in TiKV. Helps diagnose slow backup performance and identify bottlenecks during cluster backup processes, which is critical for ensuring backup SLAs are met.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 2, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Generation Throughput", "group": "", "used_for": "Used for monitoring the rate at which TiKV generates SST (Sorted String Table) files during backup operations, measured in bytes per second. Essential for tracking backup performance, identifying throughput bottlenecks, and ensuring backup jobs complete within expected timeframes.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_backup_range_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 65, "dashboard": "Dedicated Dataplane Node Info", "title": "Write Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the write throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify write bandwidth bottlenecks, saturation issues, or abnormal I/O patterns that could impact storage performance and application workloads.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 63, "dashboard": "Dedicated Dataplane Node Info", "title": "Read IO Latency", "group": "EBS Volume Performance", "used_for": "Used for monitoring the 99th percentile and maximum read I/O latency of AWS EBS volumes attached to dedicated dataplane nodes. Critical for diagnosing storage performance bottlenecks and identifying when EBS volume read operations are experiencing high latency that could impact TiKV read performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "__rate_interval", "instance" ], "promql": [ "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 72, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the average disk read wait latency on TiKV nodes at the EBS volume level. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk reads in cloud-deployed TiKV instances.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 90, "dashboard": "Dedicated Dataplane Node Info", "title": "EBS Volume IO Exceeded ", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which AWS EBS volumes exceed their provisioned IOPS and throughput limits at the cluster level. Critical for identifying storage bottlenecks caused by hitting EBS performance caps, which can lead to degraded database performance and increased latency.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "__rate_interval" ], "promql": [ "rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{}[$__rate_interval])", "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 41, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "", "used_for": "Used for monitoring the latency distribution when creating connections to external storage systems (like S3, GCS) during backup and import operations in TiKV. Critical for diagnosing slow backup/restore initialization issues and external storage connectivity problems.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 43, "dashboard": "Backup & Import-Cluster-Clinic", "title": "External Storage Create Duration", "group": "", "used_for": "Used for monitoring the p99 latency of external storage connection initialization during backup and restore operations. Critical for diagnosing slow backup/restore startup issues caused by delays in establishing connections to external storage systems like S3 or GCS.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,type))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 67, "dashboard": "Dedicated Dataplane Node Info", "title": "Read Throughput", "group": "EBS Volume Performance", "used_for": "Used for monitoring the read throughput (bytes per second) of EBS volumes attached to dedicated dataplane nodes. Helps identify I/O bottlenecks, diagnose storage performance issues, and verify whether read-intensive workloads are approaching EBS volume throughput limits.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance" ], "promql": [ "rate(aws_ebs_csi_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 12, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze coprocessor requests in TiKV during backup and import operations. Critical for diagnosing performance bottlenecks when TiDB performs table checksums for data validation or statistics collection during backup/restore workflows.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 6, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Backup SST Duration", "group": "", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file backup operations in TiKV, showing p95, p99, and average durations. Critical for diagnosing backup performance issues and identifying slow backup ranges that may impact backup completion time or cluster performance during backup operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 76, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiKV)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait time for disk write operations on TiKV nodes' EBS volumes. Essential for identifying storage I/O bottlenecks and diagnosing performance degradation caused by slow disk writes at the infrastructure level.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 13, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes or indicate storage performance degradation.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 56, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Thread Count", "group": "", "used_for": "Used for monitoring the number of active SST import threads in TiKV instances during data import operations. Helps diagnose import performance issues and verify that import parallelism is properly configured when restoring backups or importing large datasets.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 77, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk read wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average read wait latency of EBS volumes serving TiFlash nodes in dedicated dataplane deployments. Essential for identifying storage I/O bottlenecks and diagnosing slow query performance caused by disk read delays at the cluster level.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 27, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Errors", "group": "", "used_for": "Used for monitoring the rate of import errors occurring in TiKV during data import operations. Essential for quickly identifying and diagnosing import failures, data ingestion issues, or problems with backup restoration processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(tikv_import_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 79, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk wait latency", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average wait latency per I/O operation on EBS volumes in dedicated dataplane nodes, tracking both read and write operations separately. Essential for diagnosing storage performance bottlenecks and identifying when disk I/O latency impacts cluster performance, particularly useful when investigating slow queries or high storage layer delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)", "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 55, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import CPU Utilization", "group": "", "used_for": "Used for monitoring CPU utilization of SST (Sorted String Table) import threads in TiKV during data import operations. Essential for diagnosing performance bottlenecks during backup restoration or bulk data loading, helping identify whether import processes are CPU-bound or if thread resources are properly utilized.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 78, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk write wait latency (TiFlash)", "group": "EBS Volume Performance - Cluster latitude", "used_for": "Used for monitoring the average disk write wait latency for TiFlash nodes at the cluster level. Helps identify storage I/O bottlenecks and disk performance degradation that could impact TiFlash's columnar storage operations and analytical query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "k8s_cluster", "namespace" ], "promql": [ "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(node)\nlabel_replace(\n kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n \"instance\", \"$1\", \"node\", \"(.*)\"\n)" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 19, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes and determining if disk subsystems are saturated.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 49, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import RPC Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of import RPC requests in TiKV during data import operations. Critical for diagnosing performance bottlenecks and slow import operations when restoring backups or bulk loading data into the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, request))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 73, "dashboard": "Dedicated Dataplane Node Info", "title": "Disk Bandwidth", "group": "EBS Volume Performance - Cluster latitude", "used_for": "**Used For:**\n\nUsed for monitoring disk read and write bandwidth across EBS volumes in the dedicated dataplane cluster. Essential for identifying disk I/O bottlenecks, detecting abnormal storage traffic patterns, and ensuring EBS volumes are not approaching their throughput limits during heavy workloads.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster" ], "promql": [ "rate(node_disk_read_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])", "rate(node_disk_written_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 89, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP Connections", "group": "", "used_for": "Used for monitoring the current number of established TCP connections on dedicated dataplane nodes. Helps identify connection leaks, abnormal connection spikes, or resource exhaustion issues that could impact network communication and service availability.", "component": "", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance" ], "promql": [ "node_netstat_Tcp_CurrEstab{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 398, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with source database changes in real-time.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 59, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Wait Duration", "group": "", "used_for": "Used for monitoring the wait time in the import queue during data import operations in TiKV. Helps diagnose import bottlenecks and queue congestion when backup restoration or bulk data loading is slower than expected.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 468, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring downstream systems receive near real-time updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 469, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "", "used_for": "Used for monitoring the rate of event feed errors per minute in TiCDC's KV client, categorized by error type. Critical for identifying replication issues, connection problems, or data synchronization failures that could impact downstream data consistency.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 87, "dashboard": "Dedicated Dataplane Node Info", "title": "Segments retransmitted", "group": "", "used_for": "Used for monitoring the rate of TCP segment retransmissions on dedicated dataplane nodes. Helps diagnose network reliability issues, packet loss, or congestion problems that may impact cluster communication and overall performance.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "rate(node_netstat_Tcp_RetransSegs{instance=~\"$instance\"}[1m])" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 46, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Write/Download RPC Duration", "group": "", "used_for": "Used for monitoring the latency distribution of import write and download RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when importing large datasets or restoring backups, helping identify whether slow imports are caused by write or download operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 47, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "", "used_for": "Used for monitoring the latency distribution of ingest RPC requests during data import operations in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets, helping identify slow ingest operations that may delay backup restoration or bulk data loading.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 50, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Rewrite SST Duration", "group": "", "used_for": "Used for monitoring the latency of SST file rewriting operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing backup data, particularly when SST files need to be rewritten to match the target cluster's key ranges.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 53, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations during data import in TiKV. Critical for diagnosing performance bottlenecks when importing large datasets, such as during backup restoration or bulk data loading operations.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 31, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Download SST Throughput", "group": "", "used_for": "Used for monitoring the throughput of SST (Sorted String Table) file downloads during data import operations in TiKV. Helps diagnose import performance bottlenecks and network bandwidth issues when restoring backups or importing large datasets.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "rate(tikv_import_download_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 52, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Read SST Duration", "group": "", "used_for": "Used for monitoring the latency distribution of SST file read operations during data import in TiKV. Critical for diagnosing slow import performance and identifying bottlenecks in the SST file download/read phase of backup restoration or data migration processes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_download_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 466, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD operator/m", "group": "", "used_for": "Used for monitoring the rate of PD scheduling operators created per minute, broken down by operator type. Helps identify scheduling activity patterns and diagnose issues related to region balancing, leader transfers, and other cluster rebalancing operations in TiDB.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 479, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiDB query duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile query duration in TiDB, excluding internal queries and simple Show/Use statements. Critical for identifying tail latency issues and detecting slow query outliers that may impact user experience, particularly useful when diagnosing performance degradation in TiCDC replication scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[2m])) by (le, sql_type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 29, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Region", "group": "", "used_for": "Used for monitoring the total number of Raft regions on each TiKV instance during backup and import operations. Helps identify region distribution imbalances and potential hotspots that could impact backup/restore performance or cause cluster instability.", "component": "", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 473, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV min resolved ts lag", "group": "", "used_for": "Used for monitoring the replication lag between TiKV's resolved timestamp and the cluster's current TSO (Timestamp Oracle) in TiCDC change data capture scenarios. Critical for identifying data replication delays and diagnosing CDC pipeline bottlenecks that could impact downstream data synchronization freshness.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 407, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV leader change", "group": "", "used_for": "Used for monitoring the rate of Raft leader region changes in TiKV instances over 2-minute intervals. Helps identify cluster instability, frequent leader transfers, or potential network issues that cause excessive leader switching, which can impact write performance and cluster stability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 85, "dashboard": "Dedicated Dataplane Node Info", "title": "TCP In Use", "group": "", "used_for": "Used for monitoring the number of TCP sockets currently in use on dedicated dataplane nodes. Helps identify socket exhaustion issues, connection leaks, or abnormal connection patterns that could impact network communication and service availability.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "node_sockstat_TCP_inuse{instance=~\"$instance\"}" ], "dashboard_file": "dedicated-dataplane-node-info.json" }, { "panel_id": 54, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Import Ingest SST Bytes", "group": "", "used_for": "Used for monitoring the rate of SST (Sorted String Table) file ingestion during data import operations in TiKV. Critical for tracking import throughput and diagnosing slow data loading performance during backup restoration or bulk data import scenarios.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_ingest_byte{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 20, "dashboard": "PD-Cluster-Clinic", "title": "Current peer count", "group": "", "used_for": "Used for monitoring the total number of region peers currently managed by the PD cluster. Essential for tracking cluster data distribution health and identifying issues with region replication or scheduling anomalies.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 401, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scan activity patterns and diagnose potential bottlenecks in TiCDC's data replication process when capturing changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 58, "dashboard": "Backup & Import-Cluster-Clinic", "title": "IO Utilization", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring disk I/O utilization percentage across cluster nodes during backup and import operations. Critical for identifying I/O bottlenecks that may slow down data backup/restore processes and determining if disk subsystems are saturated.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 57, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Checksum Request Duration", "group": "", "used_for": "Used for monitoring the 99th percentile latency of checksum and analyze coprocessor requests in TiKV during backup and import operations. Critical for diagnosing performance bottlenecks when performing data integrity checks or table statistics collection, especially during backup/restore workflows.", "component": "", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[2m])) by (le,req))" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 10, "dashboard": "PD-Cluster-Clinic", "title": "Storage capacity", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity available across the entire TiDB cluster as reported by PD. Essential for capacity planning and identifying when the cluster is approaching storage limits that could impact write operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 37, "dashboard": "PD-Cluster-Clinic", "title": "Current storage used", "group": "", "used_for": "Used for monitoring the current storage utilization ratio across the TiDB cluster. Helps identify when the cluster is approaching storage capacity limits and needs scaling or data cleanup to prevent storage exhaustion issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 485, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write duration", "group": "", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC sink operations, including both transaction flush duration and DDL execution time when writing to downstream systems. Critical for diagnosing replication lag issues and identifying performance bottlenecks in the data synchronization pipeline between TiDB and target databases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace,changefeed))", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (le, namespace, changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 25, "dashboard": "Backup & Import-Cluster-Clinic", "title": "Leader", "group": "", "used_for": "Used for monitoring the number of Raft leader regions on each TiKV instance and detecting sudden leader drops. Critical for identifying leader balance issues, region scheduling problems, or potential TiKV node failures during backup and import operations when leader count drops by more than 10 in a 2-minute window.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "backup-and-import.json" }, { "panel_id": 97, "dashboard": "PD-Cluster-Clinic", "title": "Normal stores", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the total number of healthy TiKV stores currently online in the PD cluster. Essential for quickly assessing cluster health and detecting store failures or unavailability that could impact data availability and performance.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_up_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 467, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan region time/m", "group": "", "used_for": "Used for monitoring the total time TiKV spends scanning regions per minute for CDC (Change Data Capture) operations. Helps diagnose CDC performance issues and identify TiKV instances experiencing high region scan overhead during data replication.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 427, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resolved ts fail/m", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the rate of resolved timestamp advancement failures in TiKV instances, broken down by failure reason. Critical for diagnosing TiCDC replication lag issues and identifying which TiKV nodes are blocking CDC's ability to track data changes in real-time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 96, "dashboard": "PD-Cluster-Clinic", "title": "Abnormal stores", "group": "", "used_for": "Used for monitoring the health status of TiKV stores in the cluster by tracking disconnected, unhealthy, and low-space stores. Critical for quickly identifying storage node failures, network issues, or capacity problems that could impact cluster availability and data reliability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_disconnected_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_unhealth_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_low_space_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_down_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_offline_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_tombstone_count\"})", "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_slow_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 471, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV advance resovled ts/s", "group": "", "used_for": "Used for monitoring the rate at which TiKV advances resolved timestamps per second for CDC (Change Data Capture). Critical for diagnosing CDC replication lag and identifying TiKV instances that are slow to advance their resolved timestamps, which can bottleneck downstream data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) " ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 21, "dashboard": "PD-Cluster-Clinic", "title": "Number of Regions", "group": "", "used_for": "Used for monitoring the total number of leader regions across the PD cluster. Essential for tracking cluster data distribution and identifying potential region imbalances or abnormal region count changes that may indicate scheduling issues or cluster scaling events.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"leader_count\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 38, "dashboard": "PD-Cluster-Clinic", "title": "Current storage size", "group": "", "used_for": "Used for monitoring the total storage capacity currently used across the entire TiDB cluster as reported by PD. Essential for tracking storage growth trends, capacity planning, and identifying when the cluster is approaching storage limits that may require scaling.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 727, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Events Flush Lag", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the time lag between when TiCDC receives change events and when they are successfully flushed to the downstream sink. Critical for diagnosing replication delays and identifying bottlenecks in the data synchronization pipeline, especially when downstream systems cannot keep up with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.8, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1449, "dashboard": "PD-Cluster-Clinic", "title": "Offline store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store removal operations in PD when taking TiKV nodes offline. Helps track decommissioning status and estimate completion time when scaling down the cluster or replacing failed nodes.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"removing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 470, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV admin apply/s", "group": "", "used_for": "Used for monitoring the rate of successful Raft admin commands applied per second in TiKV, broken down by command type (excluding compaction). Essential for tracking cluster administrative operations like region splits, merges, and configuration changes, helping diagnose issues with region management and cluster topology adjustments.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 26, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Goroutine count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of goroutines and OS threads in TiCDC instances. Essential for detecting goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could indicate performance degradation or stability issues in the change data capture service.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 472, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader duration percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile latency of TiKV's leader checking operations in TiCDC's resolved timestamp mechanism. Critical for diagnosing tail latency issues in change data capture replication, particularly when TiCDC experiences delays in tracking data changes or leader verification becomes a bottleneck.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 433, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV check leader region count percentile", "group": "", "used_for": "Used for monitoring the 99.99th percentile of region count per check leader request in TiKV instances. Critical for identifying outlier scenarios where TiCDC or other components query an unusually large number of regions in a single check leader operation, which can indicate region distribution imbalances or inefficient scanning patterns.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 27, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Open FD count", "group": "Server", "used_for": "Used for monitoring the number of open file descriptors in TiCDC processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiCDC to fail or become unstable.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 400, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV unresolved region count", "group": "", "used_for": "Used for monitoring the number of TiKV regions that CDC has not yet resolved for change data capture. Critical for diagnosing CDC replication lag and identifying regions where change events are not being processed, which can indicate network issues, TiKV overload, or CDC synchronization problems.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 19, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring the uptime duration of TiCDC, TiKV, and PD components since their last restart. Essential for tracking service availability, identifying unexpected restarts, and verifying cluster stability after maintenance or deployments.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tikv\"})", "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"})" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 72, "dashboard": "PD-Cluster-Clinic", "title": "Region health", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the health status of regions in the PD cluster, tracking counts of regions in different states (e.g., pending-peer, down-peer, offline-peer, learner-peer). Essential for identifying region scheduling issues, replica availability problems, and cluster stability concerns that may impact data availability and performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)", "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1453, "dashboard": "PD-Cluster-Clinic", "title": "Online store progress", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the progress of store preparation operations in PD (Placement Driver). Helps track when stores are being brought online or initialized, which is critical during cluster scaling, node recovery, or initial deployment scenarios.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"preparing\"} > 0)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 55, "dashboard": "PD-Cluster-Clinic", "title": "Leader/Primary", "group": "", "used_for": "Used for identifying the current leader/primary member in the PD (Placement Driver) cluster. Essential for verifying cluster leadership status, troubleshooting split-brain scenarios, and confirming which PD instance is actively managing cluster metadata and scheduling decisions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "service_member_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 22, "dashboard": "PD-Cluster-Clinic", "title": "Placement Rules Status", "group": "", "used_for": "Used for monitoring the status of PD's placement rule manager, which controls data replica placement across the cluster. Essential for verifying that placement rules are properly configured and active, helping diagnose issues with data distribution, replica placement policies, and zone/rack awareness configurations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_rule_manager_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 23, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiCDC server memory consumption, tracking both resident memory (RSS) and Go heap allocation. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC resource allocation during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1111, "dashboard": "PD-Cluster-Clinic", "title": "Left time", "group": "", "used_for": "**Used For:**\n\nUsed for monitoring the estimated time remaining for PD cluster operations such as region scheduling, leader balancing, or other ongoing cluster adjustments. Helps operators understand when current rebalancing or migration tasks are expected to complete, enabling better planning for maintenance windows and capacity changes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1433, "dashboard": "PD-Cluster-Clinic", "title": "Store limit", "group": "Cluster", "used_for": "Used for monitoring the configured rate limits on TiKV stores, which control the maximum number of operations (like adding/removing peers) allowed per store. Essential for diagnosing scheduling bottlenecks and understanding why region rebalancing or recovery might be slower than expected.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [], "promql": [ "pd_cluster_store_limit" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 486, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink write rows/s", "group": "", "used_for": "Used for monitoring the rate of rows being written to downstream sinks by TiCDC changefeeds. Essential for tracking replication throughput and identifying bottlenecks in data synchronization to downstream databases.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_sink_txn_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed)", "sum (rate(ticdc_sink_ddl_exec_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 293, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD leader history", "group": "Server", "used_for": "**Used For:**\n\nUsed for tracking PD leader election history and identifying when leadership changes occur in the cluster. Essential for diagnosing cluster stability issues, investigating unexpected failovers, and correlating PD leader switches with performance anomalies or service disruptions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 24, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CPU usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiCDC server processes and comparing it against the configured CPU limit (GOMAXPROCS). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiCDC needs additional CPU resources to handle replication workload efficiently.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}[2m])", "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1112, "dashboard": "PD-Cluster-Clinic", "title": "Current scaling speed", "group": "", "used_for": "Used for monitoring the current scaling speed of the PD cluster, indicating how fast the cluster is performing data rebalancing and region scheduling operations. Essential for evaluating cluster rebalancing efficiency during scale-out/scale-in operations or when diagnosing slow data migration issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 103, "dashboard": "PD-Cluster-Clinic", "title": "Region label isolation level", "group": "Cluster", "used_for": "Used for monitoring the distribution of regions across different label isolation levels in PD's placement rules. Helps verify that regions are properly distributed according to configured label constraints (e.g., rack, zone, host) for high availability and fault tolerance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 113, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership", "group": "Server", "used_for": "Used for monitoring the rate of ownership changes in TiCDC, tracking how frequently the owner role transfers between TiCDC nodes. Essential for diagnosing cluster stability issues, as frequent ownership changes may indicate node failures, network problems, or configuration issues affecting TiCDC's high availability.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 139, "dashboard": "PD-Cluster-Clinic", "title": "Cluster ID", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for displaying the unique cluster ID of the PD (Placement Driver) cluster. Helps verify cluster identity and distinguish between multiple TiDB clusters in multi-cluster environments or during troubleshooting.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 116, "dashboard": "PD-Cluster-Clinic", "title": "PD scheduler config", "group": "Cluster", "used_for": "Used for monitoring the configuration status of PD schedulers in the cluster. Helps verify scheduler settings and diagnose issues related to region balancing, leader distribution, and other scheduling behaviors that may affect cluster performance.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 115, "dashboard": "PD-Cluster-Clinic", "title": "Current ID allocation", "group": "Cluster", "used_for": "Used for monitoring the current maximum ID allocation value in the PD cluster. Helps track ID allocation progress and diagnose potential ID exhaustion issues or abnormal allocation patterns that could impact cluster operations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"idalloc\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 117, "dashboard": "PD-Cluster-Clinic", "title": "Label distribution", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of placement labels across the TiDB cluster managed by PD. Helps diagnose data placement issues, verify label configuration correctness, and ensure proper data distribution according to placement rules for high availability and disaster recovery.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 4, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed table count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being replicated by each TiCDC instance within a changefeed. Helps identify table distribution imbalances across TiCDC nodes and diagnose replication coverage issues when troubleshooting data synchronization problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's checkpoint timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays and ensuring changefeeds are keeping up with real-time changes, helping diagnose replication bottlenecks or downstream processing issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 642, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture tables count", "group": "Changefeed", "used_for": "Used for monitoring the number of tables being captured and replicated by each TiCDC changefeed. Helps identify table distribution imbalances across changefeeds and diagnose replication coverage issues when tables are unexpectedly missing from capture.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 86, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint", "group": "Changefeed", "used_for": "Used for monitoring TiCDC changefeed replication progress by comparing the checkpoint timestamp against the current PD cluster TSO and barrier timestamp. Essential for identifying replication lag, detecting stuck changefeeds, and diagnosing data synchronization delays in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_owner_barrier_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1427, "dashboard": "PD-Cluster-Clinic", "title": "CPU Usage", "group": "Cluster", "used_for": "Used for monitoring CPU utilization of PD (Placement Driver) instances against their configured CPU limits. Essential for identifying PD performance bottlenecks, resource contention, and determining if CPU quota adjustments are needed to maintain cluster metadata management efficiency.", "component": "PD", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}[2m])", "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 110, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Ownership history", "group": "Server", "used_for": "Used for tracking TiCDC owner role changes and leadership transfers across instances. Helps diagnose cluster stability issues, frequent leader elections, or ownership handoff problems that could impact replication continuity.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) > BOOL 0.5" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 513, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Changefeed", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster. Critical for identifying data synchronization delays in changefeeds, helping diagnose slow replication issues and ensuring downstream systems receive timely data updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1430, "dashboard": "PD-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring how long each PD instance has been running since its last restart. Essential for tracking PD stability, identifying recent restarts or crashes, and correlating service disruptions with specific PD node failures during troubleshooting.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 399, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table rebalance", "group": "", "used_for": "Used for monitoring table redistribution activity across TiCDC instances during rebalancing operations. Helps identify when tables are being moved between capture nodes, which is essential for diagnosing changefeed stability issues and understanding load distribution changes in the TiCDC cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1429, "dashboard": "PD-Cluster-Clinic", "title": "Memory Usage", "group": "Cluster", "used_for": "Used for monitoring PD (Placement Driver) memory consumption, including process resident memory, Go heap system memory, and heap in-use memory. Essential for detecting memory leaks, identifying OOM risks, and diagnosing PD stability issues caused by excessive memory usage.", "component": "PD", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 603, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Difference of resolvedTs and checkpointTs", "group": "Changefeed", "used_for": "Used for monitoring the replication lag in TiCDC changefeeds by measuring the time difference between resolvedTs (latest captured transaction timestamp) and checkpointTs (latest replicated timestamp). Critical for identifying replication delays and ensuring data synchronization is keeping pace with upstream changes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "(max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed) - max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)) / 1000 > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1431, "dashboard": "PD-Cluster-Clinic", "title": "Goroutine Count", "group": "Cluster", "used_for": "Used for monitoring the number of active goroutines in PD instances. Helps identify goroutine leaks, resource exhaustion issues, or abnormal concurrency patterns that could indicate PD performance problems or potential memory issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 512, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts", "group": "Changefeed", "used_for": "Used for monitoring the replication progress of TiCDC changefeeds by comparing the resolved timestamp (the point up to which data has been replicated) against the current cluster TSO. Essential for identifying replication lag, detecting slow tables that are holding back the changefeed, and diagnosing data synchronization delays in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 163, "dashboard": "Prometheus-TiCDC-Clinic", "title": "The status of changefeeds", "group": "Changefeed", "used_for": "Used for monitoring the operational status of TiCDC changefeeds across the cluster. Essential for quickly identifying which changefeeds are running, stopped, or experiencing issues, enabling rapid troubleshooting of data replication problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 45, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator create", "group": "Operator", "used_for": "Used for monitoring the rate at which PD creates different types of scheduling operators. Essential for understanding cluster rebalancing activity and diagnosing whether PD is actively responding to region distribution imbalances or failures.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1443, "dashboard": "PD-Cluster-Clinic", "title": "Max TSO gap", "group": "Cluster", "used_for": "**Used For:**\n\nUsed for monitoring the maximum TSO (Timestamp Oracle) gap in milliseconds between different data centers in a PD cluster. Critical for identifying clock synchronization issues and potential data consistency problems in multi-datacenter deployments, where large TSO gaps can indicate network latency or PD leader switching issues.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1464, "dashboard": "PD-Cluster-Clinic", "title": "Scheduling Allowance Status", "group": "Cluster", "used_for": "Used for monitoring whether PD's scheduling operations are currently allowed or restricted in the cluster. Essential for diagnosing why region scheduling might be paused or limited, helping identify if scheduling is intentionally disabled due to maintenance, resource constraints, or automatic protection mechanisms.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 137, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Physcial", "group": "Cluster", "used_for": "Used for monitoring the current physical timestamp component of PD's global TSO (Timestamp Oracle). Essential for verifying time progression in the cluster and diagnosing TSO allocation issues or time synchronization problems that could affect transaction ordering.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 102, "dashboard": "Prometheus-TiCDC-Clinic", "title": "PD etcd requests/s", "group": "Changefeed", "used_for": "Used for monitoring the rate of etcd requests per second made by TiCDC instances to PD's etcd. Helps identify excessive etcd access patterns, potential performance bottlenecks in metadata operations, and diagnose changefeed coordination issues.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 218, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when event processing slows down or stalls.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 229, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events", "group": "Dataflow", "used_for": "Used for monitoring the number of events collected and output by TiCDC's puller component across changefeeds. Essential for tracking data replication throughput and diagnosing data flow bottlenecks or stalls in the CDC pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 514, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schema Storage GC progress", "group": "Changefeed", "used_for": "Used for monitoring the schema storage garbage collection progress in TiCDC changefeeds by comparing the GC timestamp against the cluster's current TSO (Timestamp Oracle). Helps identify schema storage cleanup lag and potential memory pressure issues when the GC timestamp falls significantly behind the cluster TSO.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_processor_schema_storage_gc_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 597, "dashboard": "Prometheus-TiCDC-Clinic", "title": "changefeed tick duration percentile", "group": "Changefeed", "used_for": "Used for monitoring the 99th and 99.9th percentile latency of TiCDC changefeed tick operations, which represent the core processing cycle for replicating data changes. Critical for identifying performance bottlenecks and replication lag issues in CDC pipelines, especially when changefeeds experience slow processing or delays in capturing and syncing data.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_owner_changefeed_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))", "histogram_quantile(0.99, sum(rate(ticdc_processor_processor_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 79, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator check", "group": "Operator", "used_for": "Used for monitoring the rate of operator checks performed by PD's scheduler across different operator types. Helps diagnose scheduling activity levels and identify which types of operators are being evaluated most frequently, useful for understanding PD's scheduling behavior and potential bottlenecks.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"check\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 83, "dashboard": "PD-Cluster-Clinic", "title": "Store capacity", "group": "Statistics - balance", "used_for": "Used for monitoring the total storage capacity of each TiKV store in the cluster. Essential for capacity planning and identifying stores approaching their storage limits before they impact cluster availability.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_capacity\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 78, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator timeout", "group": "Operator", "used_for": "Used for monitoring the number of PD scheduling operators that have timed out within a 2-minute window, categorized by operator type. Critical for identifying scheduling bottlenecks and diagnosing why region rebalancing or leader transfers are failing to complete in the expected timeframe.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"timeout\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 80, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator replaced or canceled", "group": "Operator", "used_for": "Used for monitoring the rate of PD scheduling operators that are canceled or replaced, broken down by operator type. Helps diagnose scheduling instability, conflicting operations, or frequent strategy changes that may indicate cluster rebalancing issues or configuration problems.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"cancel\"}[2m])) by (type)", "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"replace\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 77, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operator finish", "group": "Operator", "used_for": "Used for tracking the completion rate of different types of scheduling operators in PD over 2-minute intervals. Helps administrators monitor scheduling activity effectiveness and verify that balance operations, region migrations, and other cluster adjustments are completing successfully rather than timing out or failing.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"finish\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 228, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component per second. Essential for tracking data flow throughput and identifying bottlenecks in the change data capture pipeline that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 82, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Exit error count/m", "group": "Changefeed", "used_for": "Used for monitoring the rate of TiCDC processor exits caused by errors per minute across changefeeds. Critical for detecting changefeed failures and stability issues that require immediate attention to prevent data replication interruptions.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance, namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1444, "dashboard": "PD-Cluster-Clinic", "title": "Current TSO Datetime", "group": "Cluster", "used_for": "Used for displaying the current timestamp oracle (TSO) value as a datetime in the PD cluster. Helps verify TSO generation is functioning correctly and can be used to detect time synchronization issues or TSO service disruptions that would impact transaction ordering across the distributed system.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 220, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sorter output events", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output from TiCDC's sorter component across changefeeds. Essential for diagnosing data replication throughput issues and identifying bottlenecks in the event processing pipeline when replication lag occurs.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, instance, changefeed, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 47, "dashboard": "PD-Cluster-Clinic", "title": "Schedule operators count by state", "group": "Operator", "used_for": "**Used For:**\n\nUsed for monitoring the rate of change in PD scheduler operator counts grouped by state/event over 2-minute intervals. Helps diagnose scheduling activity patterns and identify whether operators are being created, completed, or timing out, which is essential for troubleshooting cluster rebalancing issues and understanding PD's scheduling behavior.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1455, "dashboard": "PD-Cluster-Clinic", "title": "99% operator region size", "group": "Operator", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile size of regions being processed by PD operators. Helps identify whether operators are handling unusually large regions, which can indicate region merge/split issues or unbalanced data distribution affecting scheduling performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type,le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 219, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of events being output by TiCDC's mounter component, which decodes raw KV changes into structured row-level events. Essential for tracking data processing throughput in the CDC pipeline and identifying bottlenecks in the event transformation stage.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 81, "dashboard": "PD-Cluster-Clinic", "title": "Operator step finish duration", "group": "Operator", "used_for": "Used for monitoring the P95 and P99 latency of PD operator step completion during cluster scheduling operations. Essential for diagnosing slow scheduling performance and identifying which operator types (like region transfers or leader elections) are experiencing delays in execution.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 664, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of rows being flushed to downstream sinks per second in TiCDC. Essential for tracking data replication throughput and identifying bottlenecks in the change data capture pipeline when data flow slows or stalls.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 67, "dashboard": "PD-Cluster-Clinic", "title": "Operator finish duration", "group": "Operator", "used_for": "Used for monitoring the p95 and p99 latency of PD operator completion times by operator type. Essential for diagnosing scheduling performance issues and identifying which operator types (like region merge, leader transfer, or replica addition) are experiencing delays in execution.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))", "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 221, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of data change events output by TiCDC's table sink to downstream systems. Essential for tracking data replication progress, verifying event delivery rates, and diagnosing replication pipeline bottlenecks or stalls in CDC workflows.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 719, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Quota", "group": "Sink - General", "used_for": "Used for monitoring memory quota allocation and redo event cache usage in TiCDC's sink manager. Critical for diagnosing memory pressure issues, preventing OOM errors, and ensuring the changefeed has sufficient memory resources to handle data replication workloads.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sinkmanager_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", component!=\"\", instance=~\"$ticdc_instance\"}", "ticdc_sinkmanager_redo_event_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 612, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Count (per second)", "group": "Sink - General", "used_for": "Used for monitoring the rate of rows being written to downstream sinks by TiCDC changefeeds per second. Essential for tracking replication throughput and identifying bottlenecks or performance degradation in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 84, "dashboard": "PD-Cluster-Clinic", "title": "Store available ratio", "group": "Statistics - balance", "used_for": "Used for monitoring the ratio of available storage capacity to total capacity for each TiKV store. Critical for identifying stores approaching capacity limits and diagnosing unbalanced storage distribution that could impact cluster stability and data placement.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_capacity\"}) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 224, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Mounter output events", "group": "Dataflow", "used_for": "Used for monitoring the total number of row change events output by TiCDC's mounter component, which parses TiKV change data into structured events. Essential for tracking data replication throughput and diagnosing pipeline bottlenecks when changefeed lag increases or replication stalls occur.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 91, "dashboard": "PD-Cluster-Clinic", "title": "Store available", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the number of available stores in the TiKV cluster from PD's perspective. Essential for diagnosing cluster capacity issues, identifying unavailable stores that may impact data placement, and verifying store health before performing balance operations or troubleshooting data distribution problems.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 64, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's scheduler worker threads, which handle transaction scheduling and conflict detection. Essential for diagnosing transaction processing bottlenecks and identifying instances with high scheduling overhead that may impact write performance.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1426, "dashboard": "PD-Cluster-Clinic", "title": "Schedulers reach limit", "group": "Operator", "used_for": "Used for monitoring when PD schedulers hit their configured limits and cannot create new operators. Critical for diagnosing why region scheduling stops or slows down, helping identify whether operator limits or per-store limits are constraining cluster rebalancing and data migration.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,name)", "sum(delta(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (desc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5939, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import Ingest RPC Duration", "group": "Backup & Import", "used_for": "Used for monitoring the latency distribution of import ingest RPC operations in TiKV during data import processes. Critical for diagnosing performance bottlenecks when ingesting SST files during backup restoration or bulk data import operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(rate(tikv_import_rpc_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1908, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether storage read threads are saturated or unevenly loaded across instances.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 40, "dashboard": "PD-Cluster-Clinic", "title": "Store leader score", "group": "Statistics - balance", "used_for": "Used for monitoring the leader score of each TiKV store, which indicates the store's capacity to host region leaders based on its load and available resources. Essential for diagnosing leader distribution imbalances and understanding why PD schedules leaders to or away from specific stores.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_score\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 665, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink flush rows", "group": "Dataflow", "used_for": "Used for monitoring the total number of rows flushed to the downstream sink by TiCDC changefeeds. Essential for tracking data replication throughput and identifying changefeeds with abnormal or stalled data flow to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 223, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table sink output events/s", "group": "Dataflow", "used_for": "Used for monitoring the rate of data change events being written to downstream tables by TiCDC's table sink component. Essential for tracking replication throughput and identifying bottlenecks in the data synchronization pipeline when events are not being delivered at expected rates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 41, "dashboard": "PD-Cluster-Clinic", "title": "Store Region score", "group": "Statistics - balance", "used_for": "**Used For:**\n\nUsed for monitoring the region distribution score across TiKV stores to evaluate load balance quality. Essential for diagnosing uneven region distribution issues and tracking the effectiveness of PD's balance-region-scheduler in redistributing regions across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_score\"}", "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}", "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 56, "dashboard": "PD-Cluster-Clinic", "title": "Store leader size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of Raft leader regions on each TiKV store. Essential for diagnosing leader distribution imbalances that can cause hotspots and uneven load across the cluster.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 57, "dashboard": "PD-Cluster-Clinic", "title": "Store Region size", "group": "Statistics - balance", "used_for": "Used for monitoring the total size of regions stored on each TiKV store in the cluster. Essential for identifying storage imbalance issues and determining if region distribution needs rebalancing across stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 721, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Event Cache Hit Ratio", "group": "Sink - General", "used_for": "Used for monitoring the cache hit ratio of redo log events in TiCDC's sink manager. Helps diagnose redo log performance issues by showing how effectively events are being served from cache versus requiring disk I/O, with low ratios indicating potential memory pressure or inefficient caching.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", type=\"hit\"}[2m])) /\nsum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4287, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiKV's unified read pool threads across instances. Essential for diagnosing read performance bottlenecks and identifying instances with high CPU consumption during read-heavy workloads.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified_read_po.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 90, "dashboard": "PD-Cluster-Clinic", "title": "Store used", "group": "Statistics - balance", "used_for": "Used for monitoring the actual disk space consumed by each TiKV store in the cluster. Essential for tracking storage utilization trends, identifying stores approaching capacity limits, and diagnosing uneven data distribution that may require rebalancing.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_used\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 613, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output DDL Count & Executing Duration", "group": "Sink - General", "used_for": "Used for monitoring TiCDC's DDL replication performance, tracking both the rate of DDL operations being executed at the sink and their p999 execution latency. Critical for diagnosing DDL replication bottlenecks and identifying when schema changes are causing downstream synchronization delays.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 614, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Sink Error Count (1min)", "group": "Sink - General", "used_for": "Used for monitoring the rate of sink execution errors in TiCDC changefeeds over a 1-minute window. Critical for detecting data replication failures and identifying problematic changefeeds that are failing to write data to downstream systems.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 85, "dashboard": "PD-Cluster-Clinic", "title": "Size amplification", "group": "Statistics - balance", "used_for": "Used for monitoring the storage amplification ratio in TiKV stores by comparing region size to available disk space. Helps identify stores with inefficient space utilization, excessive data fragmentation, or compaction issues that may lead to premature disk capacity exhaustion.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_used\"}) by (address, store) * 2^20" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 611, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Large Row (size >= 128K) Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile size of large rows (≥128KB) being replicated by TiCDC to downstream sinks. Critical for identifying oversized row issues that can cause replication bottlenecks, memory pressure, or downstream write failures in change data capture pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_large_row_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_large_row_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sink_large_row_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 610, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Output Row Batch Size", "group": "Sink - General", "used_for": "Used for monitoring the 99.9th percentile and average number of rows batched together in TiCDC sink output operations. Critical for diagnosing sink throughput efficiency and identifying whether batch sizes are optimal or if changefeeds are experiencing batching issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sinkv2_batch_row_count_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 69, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiKV instances. Essential for identifying storage engine CPU bottlenecks and diagnosing performance issues related to data read/write operations in the underlying RocksDB layer.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1463, "dashboard": "PD-Cluster-Clinic", "title": "Store witness count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of witness replicas on each TiKV store in the cluster. Helps verify witness replica distribution balance and diagnose placement issues when using witness replicas for disaster recovery configurations.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"witness_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 620, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Full Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction flush operations in TiCDC's sink component. Critical for diagnosing downstream write performance bottlenecks and identifying when transaction batches take unusually long to commit to the target database.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2531, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's garbage collection (GC) worker threads per instance. Essential for diagnosing GC performance issues, identifying instances with excessive GC overhead, and detecting potential bottlenecks in MVCC version cleanup operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "BackGround Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV background worker threads (like compaction, cleanup, and maintenance tasks). Essential for identifying when background operations consume excessive CPU resources, potentially impacting foreground query performance or indicating storage engine bottlenecks.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 634, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Input Rows", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the rate of rows being processed by TiCDC's transaction sink workers across different changefeeds. Essential for tracking data replication throughput and identifying bottlenecks in the transaction sink pipeline when rows are not being consumed as expected.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 78, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's coprocessor threads across different priority levels (normal, high, and low). Essential for diagnosing performance bottlenecks when TiDB pushes down computation to TiKV, helping identify whether coprocessor requests are consuming excessive CPU resources or experiencing priority-based contention.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 59, "dashboard": "PD-Cluster-Clinic", "title": "Store Region count", "group": "Statistics - balance", "used_for": "Used for monitoring the distribution of regions across different TiKV stores in the cluster. Essential for identifying region imbalance issues that can lead to hotspots and uneven load distribution, helping diagnose performance bottlenecks caused by poor data placement.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 50, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's leader distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write region leaders across TiKV stores. Helps identify unbalanced write hotspots and leader placement issues that can cause performance bottlenecks or overload specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 51, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot write", "used_for": "Used for monitoring the distribution of hot write regions across TiKV stores from a peer perspective. Helps identify uneven hot region distribution and potential write hotspot imbalances that could lead to performance bottlenecks on specific stores.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_peer\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 58, "dashboard": "PD-Cluster-Clinic", "title": "Store leader count", "group": "Statistics - balance", "used_for": "Used for monitoring the number of Region leaders distributed across each TiKV store in the cluster. Essential for diagnosing leader imbalance issues that can cause uneven load distribution and identifying whether PD's leader scheduling is working effectively.", "component": "PD", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_count\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 619, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the utilization ratio of TiCDC transaction sink workers, indicating how busy the workers are processing and flushing transactions to downstream. Essential for identifying worker saturation issues and determining if additional worker capacity is needed to handle replication workload efficiently.", "component": "TICDC", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) /sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance,id) *100" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 635, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backend Flush Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of DML batch commits to the downstream database in TiCDC's transaction sink. Critical for diagnosing slow replication issues and identifying backend database performance bottlenecks that affect change data capture throughput.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 691, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Backup Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV backup worker threads responsible for handling backup operations. Essential for diagnosing backup performance issues, identifying CPU bottlenecks during backup tasks, and ensuring backup operations don't consume excessive resources that could impact normal cluster operations.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[2m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 618, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Conflict Detect Duration", "group": "Sink - Transaction Sink", "used_for": "Used for monitoring the 99.9th percentile latency of transaction conflict detection in TiCDC's transaction sink, along with average detection time and queue duration. Critical for diagnosing replication delays caused by transaction conflicts when TiCDC writes to downstream databases, helping identify whether slowdowns stem from conflict resolution overhead or queue congestion.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,changefeed,instance))", "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (changefeed,instance)", "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 62, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CDC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiCDC (Change Data Capture) worker threads and TSO threads in TiKV instances. Essential for diagnosing CDC replication performance issues and identifying whether CDC workloads are causing CPU bottlenecks on specific TiKV nodes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 692, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Import CPU", "group": "Thread CPU", "used_for": "**Used For:**\n\nUsed for monitoring CPU usage of SST (Sorted String Table) import threads in TiKV instances. Essential for diagnosing performance issues during data import operations, snapshot ingestion, or when TiKV is receiving large amounts of data through bulk load processes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1465, "dashboard": "PD-Cluster-Clinic", "title": "Total written bytes on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total write throughput (in bytes) on hot leader Regions across stores. Essential for identifying write hotspots, unbalanced write loads, and potential performance bottlenecks caused by concentrated write traffic on specific leader Regions.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_bytes_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 60, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "CDC endpoint CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of CDC (Change Data Capture) endpoint threads in TiKV instances. Essential for diagnosing CDC performance bottlenecks and identifying instances experiencing high replication workload or resource contention.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cdc_.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9962, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TSO Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TSO (Timestamp Oracle) worker threads in TiKV instances. Essential for diagnosing timestamp allocation performance issues and identifying CPU bottlenecks in the TSO service that could impact transaction processing across the cluster.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"tso_worker\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 630, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Latency", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the latency of Kafka producer requests in TiCDC's message queue sink. Critical for diagnosing replication delays and identifying performance bottlenecks when TiCDC writes change data to Kafka topics.", "component": "TICDC", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572511, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raftlog fetch Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiKV's raftlog fetch worker threads, which handle retrieving Raft logs from storage. Essential for diagnosing performance bottlenecks when followers are catching up with leaders or during log replication, helping identify if log fetching is CPU-constrained.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftlog_fetch.*\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6987, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker actions", "group": "TTL", "used_for": "Used for monitoring the rate of Time-To-Live (TTL) checker actions in TiKV, broken down by action type. Helps diagnose TTL-related I/O activity and identify issues with expired data cleanup operations in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(rate(tikv_ttl_checker_actions{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 653, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC workers sending messages to message queue (MQ) sinks like Kafka. Critical for diagnosing tail latency issues in data replication to downstream MQ systems and identifying potential bottlenecks in the CDC pipeline's message delivery performance.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 6985, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL check progress", "group": "TTL", "used_for": "Used for monitoring the progress of Time-To-Live (TTL) checks across TiKV instances by showing the ratio of processed regions to total regions. Helps track TTL scan completion and identify instances that are lagging behind in expired data cleanup operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(tikv_ttl_checker_processed_regions{instance=~\"$instance\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 628, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Outgoing Bytes", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the outgoing data throughput rate from TiCDC to Kafka message queues, measured in bytes per second. Essential for tracking replication bandwidth usage, identifying network bottlenecks, and ensuring data is being successfully transmitted to downstream Kafka clusters.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namesapce=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 627, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Inflight Requests", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of in-flight (pending) requests to Kafka brokers from TiCDC's MQ sink. Essential for identifying backpressure issues, network bottlenecks, or Kafka broker slowness that could cause replication lag or sink congestion.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 61, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate bytes", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring write throughput (bytes per second) across TiKV stores to identify hot write regions and unbalanced write loads. Essential for diagnosing write hotspots, evaluating store-level write distribution, and determining if PD's load balancing is effectively spreading write traffic across the cluster.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1445, "dashboard": "PD-Cluster-Clinic", "title": "Store write query", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the write query rate per store in PD's hot region scheduling. Helps identify stores experiencing high write traffic and diagnose unbalanced write load distribution across the cluster, which is critical for optimizing hot region scheduling and preventing write hotspots.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 146, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache write entry numbers", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of hot write regions cached in PD's hotspot detection system. Essential for diagnosing write hotspot issues and understanding how many regions are experiencing concentrated write traffic that may require load balancing.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 48, "dashboard": "PD-Cluster-Clinic", "title": "Total written keys on hot leader Regions", "group": "Statistics - hot write", "used_for": "Used for monitoring the total number of keys written per second on hot leader Regions across stores. Helps identify write hotspots and unbalanced write loads, which can indicate the need for Region splitting or load rebalancing to prevent performance bottlenecks.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_keys_as_leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 143, "dashboard": "PD-Cluster-Clinic", "title": "Store write rate keys", "group": "Statistics - hot write", "used_for": "**Used For:**\n\nUsed for monitoring the write throughput in keys per second for each TiKV store, including both actual store write rates and region-level aggregated write rates. Essential for identifying hot write stores, diagnosing write load imbalances across the cluster, and determining if specific stores are becoming write bottlenecks.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}", "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 629, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Request Rate", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of requests sent to Kafka brokers by TiCDC's MQ sink. Essential for tracking replication throughput to Kafka and diagnosing bottlenecks or connectivity issues in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 631, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Records Per Request", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the number of records batched per Kafka produce request in TiCDC's MQ sink. Helps evaluate batching efficiency and diagnose throughput issues—low values may indicate poor batching performance, while high values suggest optimal message aggregation.", "component": "TICDC", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 6986, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker compact duration", "group": "TTL", "used_for": "Used for monitoring the latency of TTL (Time To Live) checker compaction operations in TiKV at p95 and p99 percentiles. Essential for diagnosing performance issues with expired data cleanup and identifying when TTL compaction operations are taking longer than expected, which could impact storage efficiency and query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_ttl_checker_compact_duration_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_ttl_checker_compact_duration_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "PD-Cluster-Clinic", "title": "Selector write events", "group": "Statistics - hot write", "used_for": "Used for monitoring the rate of hot write region detection events in PD's scheduler. Helps identify when and how frequently PD detects write hotspots across stores, which is essential for diagnosing unbalanced write load distribution and evaluating hotspot scheduling effectiveness.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 709, "dashboard": "Prometheus-TiCDC-Clinic", "title": "encoder group input channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the input channel buffer size of TiCDC's encoder group when replicating data to message queue sinks. Helps identify backpressure issues and bottlenecks in the encoding pipeline when the channel fills up, indicating that encoders cannot keep pace with incoming change events.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 626, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Kafka Producer Compression Ratio", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the compression efficiency of data sent to Kafka by TiCDC, showing both average and 99th percentile compression ratios. Essential for evaluating network bandwidth utilization and identifying changefeeds with poor compression performance that may impact throughput and storage costs.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", "histogram_quantile(0.99, sum(rate(ticdc_kafka_sink_codec_compression_ratio_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1441, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot write", "used_for": "Used for monitoring the number of pending items in PD's hot write region detection queue. Helps diagnose delays in hot region identification and potential bottlenecks in PD's hot region scheduling mechanism when the queue accumulates items.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"write\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 710, "dashboard": "Prometheus-TiCDC-Clinic", "title": "encoder group output channel size", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the output channel buffer size of TiCDC's encoder group when replicating to message queue (MQ) sinks. Helps identify backpressure issues where encoded change events are accumulating faster than they can be sent to the MQ, indicating potential downstream bottlenecks or sink performance problems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 60, "dashboard": "PD-Cluster-Clinic", "title": "Hot Region's peer distribution", "group": "Statistics - hot read", "used_for": "Used for monitoring the distribution of hot read regions across TiKV stores, showing both peer and leader roles. Helps identify load imbalance issues where hot regions are concentrated on specific stores, enabling targeted rebalancing decisions to optimize read performance.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_peer\"}", "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_leader\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 711, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of MQ worker batch processing in TiCDC's message queue sink. Critical for identifying tail latency issues when TiCDC replicates data to downstream message queue systems like Kafka, helping diagnose slow batch processing and potential replication delays.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 7326, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TTL checker poll interval", "group": "TTL", "used_for": "Used for monitoring the polling interval of TiKV's TTL (Time-To-Live) checker, which determines how frequently expired data is scanned and cleaned up. Essential for diagnosing TTL cleanup performance issues and understanding the trade-off between cleanup frequency and system overhead.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance" ], "promql": [ "max(tikv_ttl_checker_poll_interval{instance=~\"$instance\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 150, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot write", "used_for": "Used for monitoring the direction and frequency of hot write region leader transfers between TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing write-intensive regions and identify stores that are frequently transferring leaders in or out to resolve write hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 731, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Message Count Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the distribution of message counts per batch in TiCDC's Kafka producer when using the kafka-go library. Helps diagnose batching efficiency and identify whether messages are being properly aggregated before sending to Kafka, which impacts throughput and network efficiency.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 712, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Batch Size Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile batch size of messages sent by TiCDC's MQ sink workers to message queues like Kafka. Critical for diagnosing message batching efficiency and identifying whether the sink is properly aggregating changes before sending, which directly impacts throughput and downstream processing performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 733, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Messages Duration Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the duration percentiles of Kafka message batching operations in TiCDC's Kafka-Go producer implementation. Critical for diagnosing message batching latency issues and identifying performance bottlenecks when replicating data changes to Kafka sinks.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_duration{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 107, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate keys", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in terms of keys per second for each TiKV store, helping identify hot read spots and load imbalances across stores. Essential for diagnosing uneven read distribution and determining if specific stores are experiencing disproportionate read key traffic that could impact cluster performance.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1396, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiKV and PD (Placement Driver). Helps diagnose cluster membership issues, region metadata synchronization problems, and potential communication failures between TiKV nodes and PD.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 148, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot write", "used_for": "Used for tracking PD's hotspot scheduling decisions for write-heavy regions, showing the direction and frequency of peer/leader movements across stores. Essential for diagnosing whether PD is actively balancing write hotspots and identifying stores that are frequently sources or destinations of hot region migrations.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1215, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate and volume of heartbeat messages between TiKV instances and PD (Placement Driver). Essential for diagnosing cluster coordination issues, detecting communication failures between TiKV and PD, and identifying heartbeat backlogs that may indicate network problems or PD overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "PD-Cluster-Clinic", "title": "Store read rate bytes", "group": "Statistics - hot read", "used_for": "Used for monitoring the read throughput in bytes per second across TiKV stores, comparing actual store-level read rates with expected hot region read rates. Essential for identifying read hotspots, diagnosing unbalanced read traffic distribution, and validating whether PD's hot region scheduling is effectively balancing read-intensive workloads across stores.", "component": "PD", "phenomenon": "throughput", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1069, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of requests sent from TiKV to PD (Placement Driver). Helps identify communication patterns between TiKV and PD, and diagnose issues when TiKV cannot properly interact with PD for metadata operations, region scheduling, or TSO allocation.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8376, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD forward status", "group": "PD", "used_for": "Used for monitoring whether TiKV's PD client requests are being forwarded to other PD nodes. Helps diagnose PD leader election issues, network connectivity problems between TiKV and PD, or misconfigurations where requests aren't reaching the correct PD leader.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_pd_request_forwarded{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1446, "dashboard": "PD-Cluster-Clinic", "title": "Store read query", "group": "Statistics - hot read", "used_for": "Used for monitoring read query rates on TiKV stores to identify hot read regions and unbalanced read traffic. Essential for diagnosing read hotspots, evaluating load distribution across stores, and determining if read traffic is concentrated on specific regions or leaders.", "component": "PD", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}", "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 7985, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD reconnections", "group": "PD", "used_for": "Used for monitoring the rate of reconnection attempts between TiKV instances and PD (Placement Driver) servers. Critical for diagnosing network instability, PD availability issues, or connection problems that could impact cluster coordination and metadata operations.", "component": "TIKV", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_pd_reconnect_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 737, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Error Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's Kafka-Go producer when replicating data to Kafka sinks. Critical for identifying replication failures, connection issues, or message delivery problems that could cause data loss or replication lag.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_err_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 735, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Producer Batch Bytes Percentile (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the maximum batch size (in bytes) of messages sent to Kafka by TiCDC's kafka-go producer library. Helps diagnose message batching efficiency and identify potential throughput bottlenecks when replicating data to Kafka sinks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(ticdc_sink_kafka_producer_batch_size{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 739, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Retry Count (Kafka-Go Only)", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of message send retries in TiCDC's Kafka sink when using the kafka-go client library. Helps diagnose Kafka connectivity issues, network instability, or broker-side problems that cause message delivery failures requiring retries.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_kafka_producer_retry_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1070, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different types of requests from TiKV to PD (Placement Driver). Essential for diagnosing communication issues between TiKV and PD, identifying slow PD operations like TSO allocation or region heartbeats that could impact cluster coordination and transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9963, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Pending TSO Requests", "group": "PD", "used_for": "Used for monitoring the number of TSO (Timestamp Oracle) requests currently pending in TiKV instances waiting for PD response. Critical for diagnosing PD service bottlenecks, network latency issues, or TSO allocation problems that can cause transaction delays across the cluster.", "component": "TIKV", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_pd_pending_tso_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 149, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot transfer leader", "group": "Statistics - hot read", "used_for": "Used for monitoring the direction and frequency of hot read region leader transfers across TiKV stores. Helps diagnose whether PD's hotspot scheduler is actively balancing read-heavy regions and identify stores that are frequently transferring leaders in or out to resolve read hotspots.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1440, "dashboard": "PD-Cluster-Clinic", "title": "Remaining items in Queue", "group": "Statistics - hot read", "used_for": "Used for monitoring the number of pending items in PD's hot read region detection queue. Helps diagnose delays in hot region identification and potential bottlenecks in PD's hot region scheduling mechanism when the queue accumulates items.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 144, "dashboard": "PD-Cluster-Clinic", "title": "Direction of hotspot move peer", "group": "Statistics - hot read", "used_for": "Used for tracking the direction and frequency of hot region peer movements during read hotspot balancing. Helps diagnose whether PD's scheduler is actively redistributing hot read regions across stores and identify which stores are receiving or losing hot region peers/leaders.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[2m]))by (store)", "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)", "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 740, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Duration Percentile", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the 99.9th percentile latency of claim check message sending operations in TiCDC's MQ sink. Critical for diagnosing tail latency issues when TiCDC uses claim check pattern to handle large messages in message queue sinks, helping identify performance bottlenecks in external storage interactions.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,changefeed,instance))", "rate(ticdc_sink_mq_claim_check_send_message_duration_sum[30s]) / rate(ticdc_sink_mq_claim_check_send_message_duration_count[30s])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 741, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Claim Check Send Message Count", "group": "Sink - MQ Sink", "used_for": "Used for monitoring the rate of claim check messages sent by TiCDC's MQ sink when using the claim check pattern for large messages. Helps identify when TiCDC is offloading oversized messages to external storage and track the frequency of this optimization mechanism.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1466, "dashboard": "PD-Cluster-Clinic", "title": "Selector read events", "group": "Statistics - hot read", "used_for": "Used for monitoring the rate of hot read region detection events in PD's scheduler. Helps identify when and how frequently PD is detecting read hotspots across stores, which is essential for diagnosing read traffic imbalance and evaluating the effectiveness of hot region scheduling.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5995, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "IO threshold", "group": "IO Breakdown", "used_for": "Used for monitoring the configured I/O rate limit thresholds in TiKV's rate limiter by operation type. Helps administrators verify I/O throttling settings and diagnose whether performance issues are caused by rate limiting constraints.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tikv_rate_limiter_max_bytes_per_sec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10366, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Inspected duration per server", "group": "PD", "used_for": "Used for monitoring the 99th percentile duration of Raft store inspection operations per TiKV instance. Critical for diagnosing performance issues in Raft consensus protocol checks and identifying instances with abnormal inspection latency that could impact cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_inspect_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10015, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the latency of write operations from TiCDC to cloud storage sinks. Critical for diagnosing performance bottlenecks in data replication to cloud storage destinations like S3 or GCS, helping identify slow writes that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 147, "dashboard": "PD-Cluster-Clinic", "title": "Hot cache read entry numbers", "group": "Statistics - hot read", "used_for": "Used for monitoring the total number of hot read regions cached in PD's hotspot detection system. Essential for understanding read hotspot distribution across stores and diagnosing whether PD is correctly identifying and tracking read-intensive regions for scheduling decisions.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10365, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store Slow Score", "group": "PD", "used_for": "Used for monitoring the slow score of each TiKV store instance, which indicates how slow a store is responding compared to others in the cluster. Essential for identifying underperforming TiKV nodes that may be experiencing disk issues, network problems, or resource contention, helping diagnose unbalanced load distribution and potential performance bottlenecks.", "component": "TIKV", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_raftstore_slow_score{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} by (instance, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 5993, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Write IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of write I/O throughput in bytes per second across TiKV instances, broken down by operation type. Essential for diagnosing write performance bottlenecks, identifying which storage operations consume the most I/O bandwidth, and detecting abnormal disk write patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"write\"}[45s]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 7225, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate Limiter Request Wait Duration", "group": "IO Breakdown", "used_for": "Used for monitoring the 99th percentile wait duration of requests throttled by TiKV's rate limiter. Critical for diagnosing performance degradation caused by I/O throttling and identifying when storage bandwidth limits are impacting request latency.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "rate(tikv_rate_limiter_request_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) / rate(tikv_rate_limiter_request_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13132, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the p99 and p999 latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying tail latency issues that may impact transaction commit times and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 714, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write Bytes/s", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the write throughput (bytes per second) of TiCDC's cloud storage sink. Essential for tracking data replication performance to cloud storage and identifying bandwidth bottlenecks or write slowdowns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (namespace, changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 715, "dashboard": "Prometheus-TiCDC-Clinic", "title": "File Count", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the total number of files generated by TiCDC's cloud storage sink for each changefeed instance. Helps track file accumulation patterns and identify potential issues with file generation rate or storage cleanup processes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10017, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Flush duration", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the time taken to flush data files to cloud storage in TiCDC's cloud storage sink. Critical for diagnosing slow data replication to cloud destinations and identifying potential bottlenecks in the file upload process.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 52, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's balance leader scheduler, which redistributes region leaders across TiKV nodes to balance load. Essential for diagnosing leader distribution imbalances and verifying that the scheduler is actively working to equalize leadership across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 46, "dashboard": "PD-Cluster-Clinic", "title": "Scheduler is running", "group": "Scheduler", "used_for": "Used for monitoring which schedulers are currently active and allowed to run in PD. Essential for verifying scheduler configuration changes and troubleshooting why certain data rebalancing or region scheduling operations may not be executing as expected.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"allow\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 86, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region movement", "group": "Scheduler", "used_for": "Used for monitoring the movement of regions during balance operations in PD's region scheduler. Helps diagnose load balancing effectiveness by showing which stores are sources (losing regions) and targets (gaining regions), essential for identifying uneven data distribution and scheduler behavior issues.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 571, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed checkpoint lag", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the replication lag between TiCDC's changefeed checkpoint and the upstream TiDB cluster's current timestamp. Critical for identifying replication delays, ensuring data freshness in downstream systems, and diagnosing changefeed performance issues or bottlenecks.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13257, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of Raft store thread operations in TiKV, tracking p99, p999, and average durations. Critical for diagnosing Raft consensus bottlenecks and identifying when the store thread becomes a performance bottleneck in the write path.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1458, "dashboard": "PD-Cluster-Clinic", "title": "Balance Hot Region scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's hot region balance scheduler, which automatically redistributes regions with high I/O load across TiKV nodes. Essential for diagnosing uneven I/O distribution issues and verifying whether the scheduler is actively balancing hot spots in the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 10019, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Sink - Cloud Storage Sink", "used_for": "Used for monitoring the busy ratio of cloud storage sink workers in TiCDC, indicating how much time workers spend actively processing data versus idle time. Essential for identifying worker saturation issues and determining if additional worker capacity is needed to handle replication workload to cloud storage.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])/10) by (namespace,changefeed,id,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5994, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read IO bytes", "group": "IO Breakdown", "used_for": "Used for monitoring the rate of read I/O throughput in TiKV instances, broken down by operation type. Essential for diagnosing read performance bottlenecks, identifying which storage operations consume the most I/O bandwidth, and detecting abnormal disk read patterns that may impact query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", op=\"read\"}[45s]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 589, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Changefeed resolved ts lag", "group": "Scheduler", "used_for": "Used for monitoring the replication lag between TiCDC's resolved timestamp and the upstream TiDB cluster for each changefeed. Critical for identifying data synchronization delays and ensuring downstream systems receive near real-time updates.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 87, "dashboard": "PD-Cluster-Clinic", "title": "Balance leader movement", "group": "Scheduler", "used_for": "Used for monitoring the movement of Raft leaders between TiKV stores during balance operations. Helps diagnose leader distribution issues and verify that PD's balance-leader scheduler is actively redistributing leaders to achieve cluster load balancing.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (source)", "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13259, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of applying Raft log entries to TiKV's state machine, tracking p99, p999, and average durations. Critical for diagnosing write path performance issues and identifying bottlenecks in the Raft apply process that can impact overall write throughput and latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_apply_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1459, "dashboard": "PD-Cluster-Clinic", "title": "Split Bucket scheduler", "group": "Scheduler", "used_for": "Used for monitoring the activity rate of PD's split-bucket scheduler, which automatically splits hot Regions into smaller buckets to improve load balancing. Essential for diagnosing uneven data distribution issues and verifying that the scheduler is actively working to distribute hotspots across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"split-bucket-scheduler\"}[5m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 548, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks", "group": "Scheduler", "used_for": "Used for monitoring the distribution and volume of scheduling tasks across different schedulers in TiCDC. Helps identify scheduling bottlenecks, uneven task distribution, and potential performance issues in the change data capture replication pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13263, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store batch wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft store threads spend waiting to batch operations before processing, tracking p99, p999, and average latencies. Critical for diagnosing Raft processing bottlenecks and identifying when store threads are delayed due to batching inefficiencies or thread contention.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 549, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks accept", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduling tasks being accepted by TiCDC changefeeds. Helps diagnose task scheduling bottlenecks and identify changefeeds that may be struggling to accept new replication tasks, which could indicate resource constraints or coordination issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 53, "dashboard": "PD-Cluster-Clinic", "title": "Balance Region scheduler", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the activity rate of PD's balance-region scheduler, which redistributes regions across TiKV stores to maintain balanced load. Essential for diagnosing uneven data distribution issues and verifying that region balancing operations are occurring as expected during cluster rebalancing or after topology changes.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m])) by (name)", "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 141, "dashboard": "PD-Cluster-Clinic", "title": "Rule checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of rule checker events in PD's scheduler, broken down by rule name. Helps diagnose placement rule violations and scheduling constraint issues that may affect data distribution and replica placement across the cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"rule_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 588, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Schedule tasks running", "group": "Scheduler", "used_for": "Used for monitoring the number of actively running scheduler tasks in TiCDC changefeeds. Helps identify scheduling bottlenecks, task accumulation issues, and abnormal task execution patterns that could impact data replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13261, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store propose wait duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time Raft proposals spend waiting in the store thread queue before being processed in TiKV. Critical for diagnosing write latency issues caused by store thread congestion or scheduling delays, especially when p99/p999 values are high, indicating that requests are queuing excessively before Raft processing begins.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 70, "dashboard": "PD-Cluster-Clinic", "title": "Replica checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of replica checker events in PD's scheduler. Helps diagnose replica placement issues, such as missing replicas, incorrect replica counts, or replica distribution problems that trigger the checker to take corrective actions.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"replica_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 545, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table count", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the total number of tables being replicated by TiCDC changefeeds. Helps identify changefeed scale, verify table distribution across the scheduler, and diagnose issues when tables are unexpectedly added or removed from replication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13265, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send to write queue duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft store thread messages to the write queue in TiKV, tracking p99, p999, and average durations. Critical for diagnosing bottlenecks in the Raft store thread's message passing pipeline and identifying when the write queue becomes a performance constraint.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 543, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Scheduler", "used_for": "Used for identifying which table ID is experiencing the slowest replication performance in TiCDC's scheduler. Helps pinpoint specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 71, "dashboard": "PD-Cluster-Clinic", "title": "Region merge checker", "group": "Scheduler", "used_for": "Used for monitoring the rate of region merge operations checked by PD's merge checker. Essential for diagnosing region merge performance issues, identifying merge bottlenecks, and understanding whether regions are being consolidated efficiently to reduce cluster overhead.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"merge_checker\"}[2m])) by (name)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13269, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write kv db end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of completing KV database writes in TiKV's Raft store thread, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying when data persistence to RocksDB is experiencing delays in the Raft replication waterfall.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 546, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Table replication state", "group": "Scheduler", "used_for": "Used for monitoring the current replication state of tables being synchronized by TiCDC changefeeds. Helps identify tables that are stuck, lagging, or experiencing replication issues, enabling quick diagnosis of data synchronization problems in CDC pipelines.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 108, "dashboard": "PD-Cluster-Clinic", "title": "Balance Direction", "group": "Scheduler", "used_for": "Used for monitoring the direction and volume of region balance operations in PD's scheduler. Helps diagnose load balancing behavior by showing which stores are sources or targets of region transfers, essential for troubleshooting uneven data distribution and identifying potential hotspots or imbalanced clusters.", "component": "PD", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, source, target)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1424, "dashboard": "PD-Cluster-Clinic", "title": "Patrol Region time", "group": "Scheduler", "used_for": "Used for monitoring the time taken by PD to patrol and check all regions in the cluster. Essential for diagnosing region health check performance and identifying when region scanning becomes slow, which can delay detection of region issues like unavailability or imbalanced distribution.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"} != 0" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13273, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of persisting Raft store metadata to disk in TiKV, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and disk I/O performance issues that affect Raft log persistence and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13267, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store before write duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency before Raft log writes are persisted to disk in TiKV's store thread, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying delays in the Raft log replication pipeline before actual disk writes occur.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1439, "dashboard": "PD-Cluster-Clinic", "title": "scatter operator event", "group": "Scatter and Splitter", "used_for": "Used for monitoring the outcome of scatter operations in PD, tracking successful, failed, and skipped scatter operator events. Essential for diagnosing region distribution issues and identifying problems with load balancing operations across the TiKV cluster.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"skip\"}[2m])) by (event)", "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fail\"}[2m])", "delta(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"success\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13277, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit but not persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the time gap between when Raft logs are committed and when they are actually persisted to disk in TiKV store threads. Critical for diagnosing write pipeline bottlenecks and identifying delays in the Raft log persistence process that could impact data durability and write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 547, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Capture tables count", "group": "Scheduler", "used_for": "Used for monitoring the number of tables assigned to each TiCDC capture node across changefeeds. Helps identify table distribution imbalances in the scheduler and diagnose potential workload skew issues that could impact replication performance.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 544, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table replication state", "group": "Scheduler", "used_for": "Used for identifying tables with the slowest replication progress in TiCDC changefeeds. Critical for diagnosing replication lag issues and pinpointing which specific tables are causing bottlenecks in data synchronization to downstream systems.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572857, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store send proposal duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the latency of sending Raft proposals from TiKV store threads to Raft threads, tracking p99, p999, and average durations. Critical for diagnosing write path bottlenecks and identifying when the store thread is slow in submitting proposals to the Raft state machine.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 110, "dashboard": "PD-Cluster-Clinic", "title": "Filter target", "group": "Scheduler", "used_for": "Used for monitoring how often PD's scheduler filters out potential target stores during region scheduling operations. Helps diagnose why certain stores are being excluded as scheduling targets, revealing issues with placement rules, label constraints, or store state problems that prevent optimal region distribution.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[2m])) by (target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"distinct-filter\"}[2m])) by (source, target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[2m])) by (source, target, type, scope)", "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[2m])) by (source, target, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 590, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table resolved ts", "group": "Scheduler", "used_for": "Used for monitoring the resolved timestamp of the slowest table in TiCDC's scheduler, compared against the cluster's current TSO. Critical for identifying replication lag bottlenecks caused by specific slow tables that are holding back the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 541, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table checkpoint", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for identifying the slowest table's replication checkpoint timestamp in TiCDC changefeeds compared to the current PD TSO. Critical for detecting replication lag bottlenecks caused by specific tables that are falling behind, helping pinpoint which tables are slowing down the overall changefeed progress.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1435, "dashboard": "PD-Cluster-Clinic", "title": "scatter store selection", "group": "Scatter and Splitter", "used_for": "Used for monitoring how PD distributes scattered regions across TiKV stores, showing both leader and follower replica placement. Essential for diagnosing load balancing issues during table scatter operations and identifying whether regions are being evenly distributed across stores.", "component": "PD", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"false\"}[2m])) by (store)", "sum(delta(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"true\"}[2m])) by (store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 109, "dashboard": "PD-Cluster-Clinic", "title": "Filter source", "group": "Scheduler", "used_for": "Used for monitoring the number of scheduling operations filtered at the source store level in PD's scheduler. Helps diagnose why certain stores are being excluded as source candidates for region scheduling, revealing constraints or issues preventing regions from being moved from specific stores.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[2m])) by (source, type, scope)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 689, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table ID", "group": "Slow Table", "used_for": "Used for identifying which table ID is experiencing the slowest replication in TiCDC changefeeds. Critical for pinpointing specific tables causing replication lag or bottlenecks, enabling targeted troubleshooting of data synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13279, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Process ready duration", "group": "Raft IO", "used_for": "Used for monitoring the time Raft proposals spend in the ready state before being processed in TiKV. Critical for diagnosing Raft processing bottlenecks and identifying delays in the consensus protocol pipeline that can impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13281, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Process ready duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile duration of Raft log entries becoming ready for processing in TiKV, broken down by server instance. Critical for diagnosing Raft replication delays and identifying specific TiKV nodes experiencing slow consensus processing that could impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "PD-Cluster-Clinic", "title": "gRPC 99% Completed commands duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC command completion in PD (Placement Driver). Critical for identifying slow gRPC operations that may indicate PD performance bottlenecks, network issues, or overload conditions affecting cluster metadata management and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (grpc_method, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1, "dashboard": "PD-Cluster-Clinic", "title": "gRPC Completed commands rate", "group": "gRPC", "used_for": "Used for monitoring the rate of completed gRPC commands per method in PD (Placement Driver). Helps identify which gRPC operations are most frequently executed and diagnose throughput bottlenecks or unusual traffic patterns in PD's API layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (grpc_method)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 691, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slowest table region count", "group": "Slow Table", "used_for": "Used for monitoring the number of regions in the slowest table being replicated by TiCDC changefeeds. Helps identify replication bottlenecks caused by tables with excessive region counts, which can impact CDC performance and indicate the need for table splitting or workload optimization.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13285, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write loop duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of the Raft store write loop on each TiKV instance. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow Raft log persistence, which directly impacts transaction commit latency.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13275, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store commit and persist duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the p99 and p999 latency of Raft log commit and persist operations in TiKV's store thread. Critical for diagnosing write path bottlenecks and identifying when disk I/O or Raft replication is causing slow writes to storage.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13271, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write end duration", "group": "Raft Waterfall", "used_for": "Used for monitoring the duration of the store write end phase in TiKV's Raft replication waterfall, tracking p99, p999, and average latencies. Critical for diagnosing write performance bottlenecks in the final stage of Raft log persistence, helping identify when store-level write operations are experiencing delays that impact overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99, 999 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_sum{instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3690, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends committing Raft logs to disk. Critical for diagnosing write path performance issues and identifying disk I/O bottlenecks that could impact transaction commit latency and overall cluster write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1462, "dashboard": "PD-Cluster-Clinic", "title": "HTTP 99% Completed commands duration", "group": "HTTP", "used_for": "Used for monitoring the 99th percentile latency of HTTP API commands handled by PD (Placement Driver). Critical for identifying slow API responses and diagnosing performance issues in PD's HTTP service layer that may impact cluster management operations.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (service, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1461, "dashboard": "PD-Cluster-Clinic", "title": "HTTP Completed commands rate", "group": "HTTP", "used_for": "Used for monitoring the rate of completed HTTP commands handled by PD servers, broken down by service type and instance. Helps identify HTTP API throughput patterns, detect traffic anomalies, and diagnose potential bottlenecks in PD's HTTP service layer.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (service,ip)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 39, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency distribution of Raft log append operations in TiKV's storage layer. Critical for diagnosing write performance bottlenecks and identifying when disk I/O or Raft replication is causing delays in transaction commits.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1422, "dashboard": "PD-Cluster-Clinic", "title": "MVCC DB total size", "group": "etcd", "used_for": "Used for monitoring the total size and actual usage of PD's embedded etcd MVCC database. Critical for tracking etcd storage growth and identifying potential space issues that could impact cluster metadata management and PD stability.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}", "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13283, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write loop duration", "group": "Raft IO", "used_for": "Used for monitoring the time spent in TiKV's store write loop, which handles persisting Raft log entries to disk. Critical for diagnosing write path bottlenecks and identifying when the store thread is overwhelmed by write operations, which can cause replication delays and affect cluster write performance.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_loop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 693, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table checkpoint ts", "group": "Slow Table", "used_for": "Used for monitoring the checkpoint timestamp lag of slow tables in TiCDC changefeeds compared to the current PD timestamp. Critical for identifying replication delays caused by slow tables and diagnosing which processing stage (e.g., puller, sorter, mounter) is causing the bottleneck in data synchronization.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the time TiKV spends applying Raft logs to the storage engine. Critical for diagnosing write path performance issues and identifying bottlenecks in the Raft log application process that could impact write throughput and latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 699, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag details", "group": "Slow Table", "used_for": "Used for monitoring replication lag details of slow tables in TiCDC changefeeds across different processing stages (resolved TS, checkpoint TS, and TiKV min resolved TS). Critical for diagnosing which specific tables are causing replication delays and identifying bottlenecks in the CDC pipeline stages.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster", "tikv_instance" ], "promql": [ "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 695, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table resolved ts", "group": "Slow Table", "used_for": "Used for monitoring the resolved timestamp progress of slow tables in TiCDC changefeeds compared to the current PD timestamp. Helps identify tables that are lagging behind in replication and diagnose which stage (scan/incremental) is causing the slowdown, critical for troubleshooting replication delays.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 560, "dashboard": "Prometheus-TiCDC-Clinic", "title": "TiKV scan tasks/m", "group": "Scheduler", "used_for": "Used for monitoring the rate of CDC (Change Data Capture) scan tasks per minute in TiKV instances. Helps identify scan workload patterns and diagnose potential bottlenecks in TiCDC's data replication process when tracking changes from TiKV.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3688, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations on each TiKV instance. Critical for diagnosing write performance bottlenecks and identifying instances experiencing slow disk I/O or Raft replication delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13376, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations across TiKV instances, including overall append duration, KV database writes, and Raft database writes. Critical for diagnosing write performance bottlenecks and identifying which TiKV servers experience slow Raft log persistence that could impact transaction commit latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_kvdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_send_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_callback_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 271, "dashboard": "Prometheus-TiCDC-Clinic", "title": "On disk data size", "group": "DB", "used_for": "Used for monitoring the total size of data stored on disk by TiCDC's sorter component per instance. Essential for tracking disk space usage during change data capture operations and identifying potential disk capacity issues or abnormal data accumulation.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5, "dashboard": "PD-Cluster-Clinic", "title": "Handle transactions rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd transaction operations handled by PD instances, broken down by success/failure status. Essential for diagnosing PD's etcd transaction throughput and identifying transaction processing bottlenecks or failures that could impact cluster metadata operations.", "component": "PD", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 703, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Slow table lag duration percentile", "group": "Slow Table", "used_for": "Used for monitoring the 99th percentile replication lag duration of slow tables in TiCDC changefeeds, tracking both resolved timestamp and checkpoint timestamp lag. Critical for identifying tables that are falling behind in data replication and diagnosing CDC performance bottlenecks that could impact downstream data freshness.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 32, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs to the state machine in TiKV instances. Critical for diagnosing write path performance issues and identifying instances experiencing slow log application that could impact data replication and write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 6, "dashboard": "PD-Cluster-Clinic", "title": "99% Handle transactions duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in PD. Critical for identifying slow transaction processing that could impact cluster metadata operations and overall cluster coordination performance.", "component": "PD", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 13380, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Write task block duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile blocking duration of Raft write tasks per TiKV instance. Critical for diagnosing write performance bottlenecks caused by Raft message queue congestion or I/O pressure in the storage layer.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 108, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile number of Raft proposals processed per ready batch in TiKV. Helps diagnose Raft apply efficiency and identify instances with abnormal proposal batching, which can indicate write throughput bottlenecks or consensus layer performance issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 273, "dashboard": "Prometheus-TiCDC-Clinic", "title": "In-memory data size", "group": "DB", "used_for": "Used for monitoring the amount of data held in memory by TiCDC's sorter component per instance. Critical for diagnosing memory pressure issues and determining if TiCDC needs memory tuning or if data is being efficiently spilled to disk during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 34, "dashboard": "PD-Cluster-Clinic", "title": "99% Peer round trip time seconds", "group": "etcd", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile network round-trip time between etcd peers in the PD cluster. Critical for diagnosing network latency issues, etcd consensus performance problems, and identifying slow peer communications that could impact PD's availability and cluster metadata operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, To, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 274, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write duration", "group": "DB", "used_for": "Used for monitoring the write latency distribution of TiCDC's sorter database operations. Essential for diagnosing performance bottlenecks in TiCDC's data sorting and buffering layer, particularly when replication lag increases or change data capture throughput degrades.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 286, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CPU usage", "group": "DB", "used_for": "Used for monitoring CPU usage of TiCDC sorter workers, broken down by worker name and instance. Essential for identifying CPU bottlenecks in the change data capture sorting process and diagnosing performance issues when TiCDC replication lags behind.", "component": "TICDC", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[2m])) by (name, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 272, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Level files", "group": "DB", "used_for": "Used for monitoring the distribution of SSTable files across different levels in TiCDC's embedded database (likely LevelDB/RocksDB). Helps diagnose compaction issues, storage health, and potential performance degradation when files accumulate in higher levels or level 0.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 7, "dashboard": "PD-Cluster-Clinic", "title": "99% WAL fsync duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile fsync latency of etcd's Write-Ahead Log (WAL) in PD. Critical for diagnosing PD performance issues caused by slow disk I/O, as high WAL fsync duration directly impacts cluster metadata operations and can lead to leader election delays or cluster instability.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 152, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk backend commit rate", "group": "etcd", "used_for": "Used for monitoring the rate of etcd backend commit operations in PD over 2-minute intervals. Essential for tracking etcd write throughput and identifying potential bottlenecks in PD's metadata storage layer that could impact cluster coordination performance.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 44, "dashboard": "PD-Cluster-Clinic", "title": "etcd disk wal fsync rate", "group": "etcd", "used_for": "Used for monitoring the rate of WAL (Write-Ahead Log) fsync operations in PD's etcd storage over 2-minute intervals. Essential for identifying write throughput patterns and detecting potential disk I/O bottlenecks that could impact PD's metadata persistence and cluster coordination performance.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "delta(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}[2m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 151, "dashboard": "PD-Cluster-Clinic", "title": "99% backend commit duration", "group": "etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd backend commit operations in PD. Critical for diagnosing PD performance issues caused by slow disk writes, as high commit latency directly impacts cluster metadata operations and can indicate storage bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 7, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft Propose", "used_for": "Used for monitoring the rate of different types of Raft proposals (local reads, normal writes, and read index requests) in TiKV. Essential for understanding read/write workload distribution and diagnosing Raft layer performance issues or imbalanced request patterns across the cluster.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 13382, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store io task reschedule", "group": "Raft IO", "used_for": "Used for monitoring the frequency of Raft IO task rescheduling in TiKV store threads, tracking both region-level reschedules and pending task reschedules. Essential for diagnosing IO thread contention and identifying when the store thread is overloaded, causing tasks to be rescheduled rather than executed immediately.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", "sum(tikv_raftstore_io_reschedule_pending_tasks_total{instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 288, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write speed", "group": "DB", "used_for": "Used for monitoring the write throughput of TiCDC's sorter database, which temporarily stores change data during replication. Essential for diagnosing TiCDC performance bottlenecks, identifying instances with high disk I/O pressure, and ensuring the sorter can keep up with upstream data changes.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 92, "dashboard": "PD-Cluster-Clinic", "title": "Raft term", "group": "etcd", "used_for": "Used for monitoring the Raft term number in PD's embedded etcd cluster. Helps identify leader elections and cluster stability issues—frequent term changes indicate leadership instability or network partitions affecting consensus.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"term\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 275, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write delay", "group": "DB", "used_for": "Used for monitoring the write delay in TiCDC's downstream database operations. Essential for diagnosing replication lag issues and identifying bottlenecks in data synchronization to the target database, helping determine if slow writes are causing CDC pipeline delays.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 277, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write bytes", "group": "DB", "used_for": "Used for monitoring the rate of bytes written to TiCDC's sorter database storage. Essential for tracking I/O write throughput and identifying potential disk bottlenecks or excessive write activity during change data capture operations.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 93, "dashboard": "PD-Cluster-Clinic", "title": "Raft committed index", "group": "etcd", "used_for": "Used for monitoring the Raft committed index in PD's embedded etcd cluster. Essential for tracking etcd consensus progress and diagnosing replication lag or split-brain issues when the committed index stops advancing or diverges across PD nodes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"committedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 119, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft read proposals (local reads and read index requests) per TiKV instance. Helps identify read traffic distribution across the cluster and diagnose imbalanced read workloads or instances experiencing abnormally high read proposal rates.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 278, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - First", "group": "DB", "used_for": "Used for monitoring the latency of the first read operation in TiCDC's sorter database iterator. Helps diagnose performance issues in TiCDC's data sorting mechanism, particularly when investigating delays in change data capture processing or replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 120, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft Propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiKV instance. Helps identify unbalanced write workloads across TiKV nodes and diagnose performance bottlenecks when certain instances handle disproportionately high proposal rates.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 279, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Write OPS", "group": "DB", "used_for": "Used for monitoring the write operations per second to TiCDC's sorter database across instances. Essential for tracking TiCDC's data processing throughput and identifying instances with abnormal write activity or performance bottlenecks in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 282, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - Next", "group": "DB", "used_for": "Used for monitoring the latency distribution of iterator \"next\" operations when reading from TiCDC's sorter database. Critical for diagnosing performance bottlenecks in TiCDC's event sorting and processing pipeline, especially when replication lag increases.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 29, "dashboard": "PD-Cluster-Clinic", "title": "PD server TSO handle time", "group": "TiDB", "used_for": "**Used For:**\n\nUsed for monitoring the latency of PD server handling TSO (Timestamp Oracle) requests at various percentiles (p90, p99, p999, p99999). Critical for diagnosing transaction performance issues, as high TSO handle times directly impact TiDB's ability to assign timestamps for distributed transactions and can indicate PD server bottlenecks or network problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [ 90, 99, 999, 99999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "histogram_quantile(0.99999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 41, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft proposals spend waiting in the queue before being processed in TiKV. Critical for diagnosing write latency issues caused by proposal queue congestion or slow Raft processing, especially during high write load scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "PD-Cluster-Clinic", "title": "Raft applied index", "group": "etcd", "used_for": "Used for monitoring the Raft applied index in PD's embedded etcd, which indicates the last log entry applied to the state machine. Essential for diagnosing etcd replication lag, consensus issues, and ensuring PD cluster members are synchronized with committed Raft log entries.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"appliedIndex\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 42, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Propose wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed on each TiKV instance. Critical for diagnosing Raft layer bottlenecks and identifying instances experiencing proposal queuing delays that impact write latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 280, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - First", "group": "DB", "used_for": "Used for monitoring the rate of first-read operations per second in TiCDC's sorter database iterator. Helps identify throughput patterns and potential bottlenecks when TiCDC initializes data reads during change data capture processing.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13524, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the wait time distribution of write tasks in TiKV's raftstore before being processed. Critical for diagnosing write path bottlenecks and identifying when the store thread is overloaded, causing write requests to queue up before execution.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 281, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read duration - Release", "group": "DB", "used_for": "Used for monitoring the latency of releasing read iterators in TiCDC's sorter database component. Helps diagnose performance bottlenecks in the sorter's resource cleanup operations, which can impact overall CDC replication efficiency when release operations become slow.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 13522, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Store write wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft store write tasks are executed on each TiKV instance. Critical for diagnosing write bottlenecks and identifying when the Raft store thread pool is overloaded or experiencing queueing delays.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 287, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - Release", "group": "DB", "used_for": "Used for monitoring the rate of read operations during sorter database iterator release in TiCDC. Helps identify potential resource cleanup issues or bottlenecks when releasing database iterators during change data capture processing.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572700, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write handle msg duration", "group": "Raft Propose", "used_for": "Used for monitoring the time spent handling write messages in TiKV's Raft store thread. Critical for diagnosing write path bottlenecks and identifying when the store thread is overloaded, which can cause write stalls and increased latency across the cluster.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_handle_msg_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 142, "dashboard": "PD-Cluster-Clinic", "title": "Handle requests duration", "group": "TiDB", "used_for": "Used for monitoring the 98th percentile latency of TiDB's requests to PD (Placement Driver), broken down by request type. Critical for diagnosing communication delays between TiDB and PD, which can indicate network issues, PD overload, or cluster coordination problems affecting query performance.", "component": "TIDB", "phenomenon": "latency", "object": "request", "percentile": [ 98 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))", "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type) / avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 15, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed count", "group": "KVClient", "used_for": "Used for monitoring the number of active EventFeed gRPC connections between TiCDC and TiKV nodes. Essential for diagnosing change data capture issues, detecting connection failures, and verifying that TiCDC is properly receiving data change events from all TiKV instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572701, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Store write trigger size", "group": "Raft Propose", "used_for": "Used for monitoring the distribution of write batch sizes that trigger store-level writes in TiKV's Raft store. Helps diagnose write batching efficiency and identify whether write operations are being triggered at optimal sizes, which impacts I/O performance and write amplification.", "component": "TIKV", "phenomenon": "io", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_store_write_trigger_wb_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 285, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Compact duration", "group": "DB", "used_for": "Used for monitoring the compaction duration of TiCDC's sorter database, which manages event ordering during data replication. Essential for diagnosing TiCDC performance degradation caused by slow database compaction operations that can impact replication latency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2535, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft log entries spend waiting in the apply queue before being applied to the state machine in TiKV. Critical for diagnosing apply bottlenecks and identifying when the apply worker threads are overloaded, which can cause write latency spikes and replication delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1447, "dashboard": "PD-Cluster-Clinic", "title": "Tso Request batch size", "group": "TiDB", "used_for": "Used for monitoring the batch size of TSO (Timestamp Oracle) requests from TiDB to PD, showing p90, p99, and average values. Helps diagnose TSO request batching efficiency and identify potential timestamp allocation bottlenecks that could impact transaction performance.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [ 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))", "sum(delta(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(delta(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 74, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat region event QPS", "group": "Heartbeat", "used_for": "Used for monitoring the rate of region heartbeat events received by PD from TiKV stores. Essential for diagnosing region scheduling issues, detecting abnormal region state changes, and identifying stores with excessive region operations that may impact cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, event)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 283, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Read OPS - Next", "group": "DB", "used_for": "Used for monitoring the rate of iterator \"next\" operations in TiCDC's sorter database per instance. Helps identify sorter read throughput and detect performance bottlenecks when TiCDC processes change data during replication.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 28, "dashboard": "PD-Cluster-Clinic", "title": "Handle requests count", "group": "TiDB", "used_for": "Used for monitoring the rate of TSO (Timestamp Oracle) requests handled by PD server. Helps identify TSO request patterns, detect abnormal spikes in timestamp allocation requests, and diagnose potential bottlenecks in distributed transaction coordination.", "component": "TIDB", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,dc)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2536, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Apply wait duration per server", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied to the state machine in TiKV. Critical for diagnosing apply thread bottlenecks and identifying instances experiencing delays in committing writes to storage, which directly impacts write latency and transaction performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1975, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft propose speed", "group": "Raft Propose", "used_for": "Used for monitoring the average rate of Raft log proposal throughput (bytes per second) across TiKV instances. Helps identify write workload intensity and potential bottlenecks in Raft consensus replication, particularly useful when diagnosing slow writes or uneven load distribution across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 140, "dashboard": "PD-Cluster-Clinic", "title": "99% Region heartbeat handle latency", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling region heartbeat requests from TiKV stores. Critical for diagnosing PD performance bottlenecks and identifying when the cluster's metadata management is experiencing delays that could impact scheduling decisions and cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 54, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster health problems that may affect region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"ok\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1976, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft Propose", "used_for": "Used for monitoring the 99th percentile duration of RocksDB performance context operations during Raft proposal processing in TiKV, including both store and apply threads. Critical for diagnosing write path bottlenecks by identifying which RocksDB internal operations (like WAL writes, memtable operations, or compactions) are causing latency spikes during Raft log replication.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1460, "dashboard": "PD-Cluster-Clinic", "title": "TiKV side heartbeat statistics", "group": "Heartbeat", "used_for": "**Used For:**\n\nUsed for monitoring heartbeat communication between TiKV nodes and PD, tracking both the rate of heartbeat messages by type and the number of pending heartbeats. Essential for diagnosing cluster coordination issues, detecting network problems between TiKV and PD, and identifying when TiKV nodes are experiencing delays in reporting their status to the placement driver.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 29, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client receive events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of change events received by TiCDC's KV client from TiKV per second, broken down by instance and event type. Essential for tracking data capture throughput and diagnosing replication lag or event processing bottlenecks in CDC pipelines.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1977, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft message wait duration", "group": "Raft Propose", "used_for": "Used for monitoring the time Raft messages spend waiting in queues before being processed in TiKV. Critical for diagnosing Raft communication bottlenecks and identifying when message processing delays are causing replication lag or consensus slowdowns.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_msg_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 188, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client gRPC stream count", "group": "KVClient", "used_for": "Used for monitoring the number of active gRPC streams between TiCDC's KV client and each TiKV store. Essential for diagnosing connection issues, detecting stream leaks, and understanding TiCDC's data replication connectivity health across the cluster.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 131, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report active", "group": "Heartbeat", "used_for": "Used for monitoring the rate of active region heartbeat reports received by PD from TiKV stores. Essential for detecting communication issues between PD and TiKV, identifying stores with heartbeat delays, and diagnosing cluster coordination problems that could impact region scheduling and metadata synchronization.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"bind\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1400, "dashboard": "PD-Cluster-Clinic", "title": "99% store heartbeat handle duration", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile latency of PD handling store heartbeat messages from TiKV nodes. Critical for diagnosing PD performance issues when stores report slow heartbeat processing, which can indicate PD overload or network problems affecting cluster health detection.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 31, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client dispatch events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of KV events dispatched by TiCDC's KV client, including event send counts and batch resolved event sizes per second. Essential for tracking TiCDC's data capture throughput and identifying bottlenecks in event processing or replication lag issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[2m])) by (instance, namespace,changefeed, type)", "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance,namespace, changefeed, table)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 7235, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Replica read lock checking duration", "group": "Raft Process", "used_for": "Used for monitoring the time spent checking locks during replica read operations in TiKV. Critical for diagnosing performance issues with follower reads and identifying lock contention that may cause read latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_replica_read_lock_check_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 177, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client scanning regions", "group": "KVClient", "used_for": "Used for monitoring the number of regions currently being scanned by TiCDC's KV client during change data capture operations. Helps diagnose CDC performance issues related to region scanning overhead and identify potential bottlenecks in data replication when the scanning count is abnormally high or stuck.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 123, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Max duration of raft store events", "group": "Raft Process", "used_for": "Used for monitoring the maximum duration of Raft store events in TiKV, including event processing and broadcast operations. Critical for diagnosing Raft consensus performance issues, identifying slow store threads, and detecting bottlenecks in region replication that could impact write latency and cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(1.0, sum(rate(tikv_broadcast_normal_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 28, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Eventfeed error/m", "group": "KVClient", "used_for": "Used for monitoring the rate of errors occurring in TiCDC's event feed per minute, categorized by error type. Critical for diagnosing data replication issues and identifying whether errors correlate with PD scheduling operations like leader transfers or region/peer changes, which can disrupt change data capture streams.", "component": "TICDC", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 64, "dashboard": "PD-Cluster-Clinic", "title": "Region schedule push", "group": "Heartbeat", "used_for": "Used for monitoring the rate at which PD pushes region scheduling commands to TiKV stores via heartbeats. Essential for diagnosing scheduling delays and identifying stores that may be slow to receive or process region scheduling operations, which can impact cluster rebalancing and data migration performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1403, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report error", "group": "Heartbeat", "used_for": "Used for monitoring errors in PD's heartbeat stream when pushing region information to TiKV stores. Critical for diagnosing communication failures between PD and TiKV that can lead to scheduling delays and cluster metadata inconsistencies.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"err\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 5, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft Process", "used_for": "Used for monitoring the rate of Raft ready operations handled by TiKV's Raftstore, broken down by operation type. Essential for diagnosing Raft consensus performance and identifying bottlenecks in log replication, snapshot handling, or other Raft state machine transitions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 133, "dashboard": "PD-Cluster-Clinic", "title": "Region heartbeat report error", "group": "Heartbeat", "used_for": "Used for monitoring failed region heartbeat reports from TiKV stores to PD. Critical for diagnosing cluster health issues, network problems, or store failures that prevent PD from receiving timely region status updates, which can impact scheduling decisions and cluster stability.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"err\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 17, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Event size percentile", "group": "KVClient", "used_for": "Used for monitoring the size distribution of change data capture events at the KV client layer in TiCDC, tracking both 95th and 99.9th percentiles. Helps identify abnormally large events that could impact replication performance, memory usage, or indicate issues with specific data change patterns.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 999 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))", "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, instance, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 97, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client batch resolved region count", "group": "KVClient", "used_for": "Used for monitoring the distribution of batch sizes when TiCDC's KV client resolves region events. Helps diagnose event batching efficiency and identify potential bottlenecks in region change data capture throughput.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572958, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer msg length distribution", "group": "Raft Process", "used_for": "Used for monitoring the size distribution of Raft messages exchanged between TiKV peers. Helps diagnose network bandwidth issues and identify abnormally large Raft messages that could impact replication performance and cluster communication efficiency.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_peer_msg_len_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1451, "dashboard": "PD-Cluster-Clinic", "title": "99% Bucket Report Interval", "group": "Heartbeat", "used_for": "Used for monitoring the 99th percentile interval between bucket heartbeat reports from TiKV stores to PD. Critical for diagnosing communication delays or network issues between TiKV and PD that could affect cluster metadata synchronization and scheduling decisions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 459, "dashboard": "Prometheus-TiCDC-Clinic", "title": "RegionCache operation/s", "group": "KVClient", "used_for": "Used for monitoring the rate of TiCDC region cache operations per second, broken down by operation type. Essential for diagnosing region metadata access patterns and identifying potential bottlenecks in TiCDC's interaction with TiKV's region cache during change data capture.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1452, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report State", "group": "Heartbeat", "used_for": "Used for monitoring the rate and status of bucket reports sent from TiKV stores to PD. Helps diagnose heartbeat communication issues and identify stores with abnormal bucket reporting behavior that could indicate network problems or store health issues.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, type,status)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1616, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft message flush operations per TiKV instance, categorized by flush reason. Helps diagnose Raft communication efficiency and identify instances with abnormal message batching behavior or network transmission issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages received by each TiKV instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster instability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 453, "dashboard": "Prometheus-TiCDC-Clinic", "title": "ResolveLock count/m", "group": "KVClient", "used_for": "Used for monitoring the rate of lock resolution operations per minute in TiCDC's KV client. Essential for diagnosing transaction conflicts and lock contention issues that may impact CDC replication latency and throughput.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 5, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Puller output events/s", "group": "KVClient", "used_for": "Used for monitoring the rate of events being output by TiCDC's puller component per second. Essential for tracking data change capture throughput and identifying bottlenecks in the event collection pipeline from TiKV to downstream systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "tidb_cluster" ], "promql": [ "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 251, "dashboard": "Prometheus-TiCDC-Clinic", "title": "KV client cached regions", "group": "KVClient", "used_for": "Used for monitoring the number of TiKV regions cached by TiCDC's KV client for each changefeed instance. Helps diagnose region cache efficiency and potential memory issues when the cached region count grows abnormally high or drops unexpectedly during data replication.", "component": "TICDC", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1402, "dashboard": "PD-Cluster-Clinic", "title": "Heartbeat stream report", "group": "Heartbeat", "used_for": "Used for monitoring the rate of successful heartbeat region messages pushed from PD to TiKV stores. Helps diagnose communication issues between PD and TiKV nodes, particularly when stores fail to receive timely region updates or experience heartbeat stream disruptions.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "store", "tidb_cluster" ], "promql": [ "sum(delta(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"ok\"}[2m])) by (address, store)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 11, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Messages", "group": "Raft Message", "used_for": "**Used For:**\n\nUsed for monitoring the rate of accepted Raft messages sent between TiKV nodes, broken down by message type. Essential for diagnosing Raft communication issues, network problems, or cluster consensus delays when investigating replication lag or region leadership instability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"accept\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1615, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft consensus messages sent from each TiKV instance to other nodes in the cluster. Helps identify network communication patterns, detect instances with abnormal message sending rates, and diagnose Raft replication issues or network bottlenecks between TiKV nodes.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 460, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Backoff percentiles", "group": "KVClient", "used_for": "Used for monitoring the 99.999th percentile backoff duration when TiCDC's KV client encounters retryable errors communicating with TiKV. Critical for identifying extreme cases of replication delays caused by backoff mechanisms during region unavailability, lock conflicts, or other transient failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, type))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1406, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Key", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of region key read operations in PD's scheduler. Helps diagnose region metadata access patterns and identify potential hotspots or bottlenecks in PD's region key reading performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 25, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Vote", "group": "Raft Message", "used_for": "Used for monitoring the rate of Raft vote messages sent by each TiKV instance during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1411, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of write traffic across regions in the PD cluster by tracking the byte size of region write operations. Helps identify write hotspots, unbalanced region load distribution, and potential scheduling issues that could impact cluster write performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 252, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimate remaining time for initialization", "group": "KVClient", "used_for": "Used for estimating how much time remains for TiCDC's initial region cache population during changefeed startup. Helps operators understand initialization progress and predict when the changefeed will be ready to begin replicating data.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 99, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event sorting operations in TiCDC's puller component. Essential for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being processed.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1412, "dashboard": "PD-Cluster-Clinic", "title": "Write Region Key", "group": "Heartbeat distribution ", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of keys written per region during PD's scheduling operations. Helps diagnose region split/merge performance and identify regions with unbalanced write workloads that may cause hotspots or scheduling bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1309, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft Message", "used_for": "Used for monitoring the rate of dropped Raft messages in TiKV, categorized by message type. Critical for diagnosing Raft communication issues, network problems, or overloaded nodes that prevent proper consensus protocol operation and can lead to region unavailability or replication lag.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"drop\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 105, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration", "group": "KVClient", "used_for": "Used for monitoring the time distribution of DDL event merge operations in TiCDC's sorter component. Critical for diagnosing DDL replication delays and identifying performance bottlenecks in the change data capture pipeline when schema changes are being synchronized downstream.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 53, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter sort duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting operations in TiCDC's puller entry sorter. Critical for diagnosing replication lag issues caused by slow DDL processing, especially when changefeeds experience delays in handling schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1409, "dashboard": "PD-Cluster-Clinic", "title": "Region Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of time intervals between consecutive region heartbeats sent to PD. Essential for detecting region heartbeat irregularities, network issues, or TiKV health problems that could indicate cluster instability or communication delays.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 70, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check split", "group": "Raft Admin", "used_for": "Used for monitoring the rate of region split checks in TiKV's Raftstore, categorized by check type. Helps diagnose region split behavior and identify potential issues with automatic region splitting when regions grow too large or experience high load.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1408, "dashboard": "PD-Cluster-Clinic", "title": "Store Heartbeat Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of heartbeat intervals between TiKV stores and PD. Helps diagnose network issues, store health problems, or PD scheduling delays when heartbeat intervals become irregular or exceed expected thresholds.", "component": "PD", "phenomenon": "latency", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1454, "dashboard": "PD-Cluster-Clinic", "title": "Bucket Report Interval", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of bucket report intervals from TiKV stores to PD. Helps diagnose heartbeat delays and communication issues between TiKV and PD, which can indicate network problems or overloaded stores affecting cluster coordination.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 77, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft Admin", "used_for": "Used for monitoring the rate of successful Raft admin commands applied in TiKV, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes, helping diagnose cluster topology changes and Raft group management issues.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572060, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split duration", "group": "Raft Admin", "used_for": "Used for monitoring the latency of load-based region split operations in TiKV at the 80th and 90th percentiles. Essential for diagnosing performance issues when TiKV automatically splits regions based on load, helping identify bottlenecks in region rebalancing and hotspot mitigation.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.90, sum(rate(tikv_load_base_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_load_base_split_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 76, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft Admin", "used_for": "Used for monitoring the rate of Raft administrative operations in TiKV, specifically configuration changes and leader transfers. Essential for diagnosing cluster rebalancing activities, troubleshooting region migration issues, and understanding the frequency of topology changes during scaling or maintenance operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 706, "dashboard": "Prometheus-TiCDC-Clinic", "title": "batch read sorter duration percentile", "group": "KVClient", "used_for": "Used for monitoring the 99.9th percentile latency of batch read operations in TiCDC's sorter component. Critical for diagnosing data replication delays and identifying performance bottlenecks in the change data capture pipeline when reading sorted events.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance,namespace,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 705, "dashboard": "Prometheus-TiCDC-Clinic", "title": "batch read sorter size percentile", "group": "KVClient", "used_for": "Used for monitoring the 99.9th percentile batch size when TiCDC's sorter reads data from storage. Helps diagnose data processing efficiency issues and identify potential bottlenecks in the change data capture pipeline when batch sizes are abnormally small or large.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[2m])) by (le,namespace,instance,changefeed))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 71, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft Admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiKV's Raftstore. Critical for identifying extreme tail latency in the split decision process, which can indicate performance bottlenecks when regions need to be divided due to size or load thresholds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 106, "dashboard": "Prometheus-TiCDC-Clinic", "title": "DDL sorter merge duration percentile", "group": "KVClient", "used_for": "Used for monitoring the p95 and p999 latency of DDL event sorting and merging operations in TiCDC's puller component. Critical for diagnosing replication lag issues caused by slow DDL event processing, especially when changefeeds experience delays in synchronizing schema changes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 999 ], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1467, "dashboard": "PD-Cluster-Clinic", "title": "Hot Degree of Bucket", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of hot bucket degrees in PD's scheduler, tracking how data hotspots are distributed across buckets over time. Essential for diagnosing load balancing issues and identifying regions with concentrated read/write traffic that may cause performance bottlenecks.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3636, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Load base split event", "group": "Raft Admin", "used_for": "**Used For:**\n\nUsed for monitoring the rate of load-based region split events in TiKV, categorized by split type. Helps diagnose whether TiKV is actively splitting regions due to high load (CPU/read/write pressure), which is essential for understanding automatic load balancing behavior and identifying hotspot mitigation activities.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum(delta(tikv_load_base_split_event[1m])) by (type)", "" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 112, "dashboard": "PD-Cluster-Clinic", "title": "Syncer index", "group": "Region storage", "used_for": "Used for monitoring the synchronization index of PD's region syncer, which tracks the progress of region metadata synchronization across PD nodes. Essential for diagnosing region information consistency issues and detecting synchronization lag that could affect cluster metadata accuracy.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"sync_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 289, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd MVCC DB total size", "group": "Etcd", "used_for": "Used for monitoring the total size of the etcd MVCC database in PD (Placement Driver). Critical for tracking metadata storage growth and identifying potential issues with etcd capacity limits or excessive metadata accumulation that could impact TiCDC's change data capture operations.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763573619, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Peer in Flashback State", "group": "Raft Admin", "used_for": "Used for monitoring the number of Raft peers currently in flashback state on each TiKV instance. Essential for tracking the progress and scope of flashback operations, helping diagnose issues when restoring the cluster to a previous point in time.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_peer_in_flashback_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 707, "dashboard": "Prometheus-TiCDC-Clinic", "title": "mounter group channel size", "group": "KVClient", "used_for": "Used for monitoring the input channel buffer size of TiCDC's mounter group, which processes raw KV change events before converting them to table-level changes. Critical for identifying backpressure issues when the channel fills up, indicating that the mounter cannot keep pace with incoming KV events from TiKV.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "namespace", "ticdc_instance", "tidb_cluster" ], "promql": [ "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 113, "dashboard": "PD-Cluster-Clinic", "title": "History last index", "group": "Region storage", "used_for": "Used for monitoring the last Raft log index in PD's region synchronization process. Helps diagnose region metadata synchronization issues and detect potential delays or inconsistencies in region state replication across PD nodes.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"last_index\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 267, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the latency distribution of TiCDC's etcd worker tick reactor operations. Essential for diagnosing performance issues in TiCDC's etcd interaction layer, particularly when experiencing delays in metadata synchronization or coordination tasks.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1407, "dashboard": "PD-Cluster-Clinic", "title": "Read Region Byte", "group": "Heartbeat distribution ", "used_for": "Used for monitoring the distribution of bytes read from regions reported via PD heartbeats. Helps diagnose read hotspots and uneven data access patterns across regions, which can indicate load imbalance or inefficient query patterns affecting cluster performance.", "component": "PD", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 1478, "dashboard": "PD-Cluster-Clinic", "title": "State of all instances", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync state across all PD instances in the cluster. Essential for verifying DR replication status and quickly identifying instances that may have fallen out of sync or entered abnormal states during cross-region disaster recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 3637, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TopN QPS exceeds threshold", "group": "Raft Admin", "used_for": "Used for monitoring when the top read query throughput in TiKV exceeds predefined thresholds. Helps identify hot read regions or specific queries causing excessive load, enabling quick detection of unbalanced traffic distribution or potential hotspot issues in the cluster.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "tikv_read_qps_topn{order=\"0\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1475, "dashboard": "PD-Cluster-Clinic", "title": "Recovered region", "group": "DR Autosync", "used_for": "Used for monitoring the number of regions that have been recovered in the DR (Disaster Recovery) auto-sync process. Critical for tracking recovery progress after a disaster event and verifying that data replication has been successfully restored across the cluster.", "component": "PD", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recovered_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 114, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd health check duration", "group": "Etcd", "used_for": "Used for monitoring the latency of TiCDC's health check operations against etcd at p95, p99, and p999 percentiles. Critical for diagnosing etcd connectivity issues and detecting when TiCDC's coordination layer experiences slow responses, which can impact change data capture reliability and cluster coordination.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 95, 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))", "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le,instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 264, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker tick reactor duration", "group": "Etcd", "used_for": "Used for monitoring the p99 and p999 latency of TiCDC's etcd worker tick reactor operations. Essential for diagnosing TiCDC replication delays caused by slow etcd interactions, particularly when investigating change feed lag or coordination issues between TiCDC nodes.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99, 999 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1474, "dashboard": "PD-Cluster-Clinic", "title": "Tick", "group": "DR Autosync", "used_for": "Used for monitoring the rate of DR (Disaster Recovery) auto-sync ticks in PD. Helps verify that the DR replication coordination mechanism is actively running and identify potential issues with DR synchronization scheduling.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=\"$instance\"}[5m])" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 256, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction execution duration in TiCDC's etcd worker. Critical for diagnosing coordination and metadata operation latency issues that could impact TiCDC's change data capture performance and replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12886, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC kv sync duration", "group": "Raft Log", "used_for": "Used for monitoring the latency of syncing Raft log entries to the underlying key-value storage engine in TiKV, tracking p95 and p99 percentiles. Critical for diagnosing write performance bottlenecks and disk I/O issues that affect Raft log persistence and cluster consensus stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 258, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker exec etcd txn duration percentile", "group": "Etcd", "used_for": "Used for monitoring the p95 and p99 latency of etcd transaction executions in TiCDC's etcd worker. Critical for diagnosing metadata operation performance issues and identifying when etcd transactions become slow, which can impact TiCDC's change data capture coordination and state management.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 12882, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write duration", "group": "Raft Log", "used_for": "Used for monitoring the write latency (p95, p99) when TiKV garbage collects Raft logs to disk. Critical for diagnosing performance issues during Raft log cleanup operations, especially when log accumulation causes storage pressure or impacts cluster stability.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12884, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC seek operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of seek operations performed during Raft log garbage collection in TiKV. Helps identify I/O overhead from log cleanup activities and diagnose performance issues related to excessive seek operations during Raft log compaction.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_seek_operations_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 12887, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log lag", "group": "Raft Log", "used_for": "**Used For:**\n\nUsed for monitoring the rate of Raft log replication lag across TiKV instances. Critical for identifying followers that are falling behind in log synchronization, which can indicate network issues, slow disk I/O, or overloaded nodes affecting data consistency and availability.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_log_lag_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1477, "dashboard": "PD-Cluster-Clinic", "title": "State ID", "group": "DR Autosync", "used_for": "Used for monitoring the current disaster recovery (DR) state identifier in PD's auto-sync replication. Essential for tracking DR state transitions and verifying the cluster's replication status during failover or recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12881, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC write operations ", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection write operations per TiKV instance. Helps identify instances with high log cleanup activity, which may indicate heavy write workloads or potential performance bottlenecks in log management.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1473, "dashboard": "PD-Cluster-Clinic", "title": "State", "group": "DR Autosync", "used_for": "Used for monitoring the current state of PD's disaster recovery (DR) auto-sync replication. Essential for tracking DR replication status and quickly identifying when the cluster transitions between sync states (e.g., sync, async, or disconnected) during disaster recovery operations.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=\"$instance\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 12974, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log GC failed", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log garbage collection failures in TiKV instances. Critical for identifying storage cleanup issues that can lead to disk space exhaustion and degraded cluster performance.", "component": "TIKV", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_failed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 254, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size", "group": "Etcd", "used_for": "Used for monitoring the distribution of etcd transaction sizes in TiCDC's etcd worker component. Helps identify abnormally large transactions that could impact etcd performance and diagnose potential issues with TiCDC's metadata operations or changefeed state management.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 290, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% Handle transactions duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile latency of etcd transaction handling in TiCDC's coordination layer. Critical for diagnosing slow metadata operations and replication coordination issues that could impact CDC changefeed performance.", "component": "TICDC", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1476, "dashboard": "PD-Cluster-Clinic", "title": "Recover progress", "group": "DR Autosync", "used_for": "Used for monitoring the disaster recovery (DR) auto-sync recovery progress in PD. Essential for tracking the completion status when recovering from a DR scenario, helping operators understand how far along the cluster is in restoring normal replication state.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "pd_replication_dr_recover_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "pd_detail.json" }, { "panel_id": 2, "dashboard": "TiDB-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring the commands per second (CPS) executed by each TiDB instance in the cluster. Helps identify uneven query distribution across instances, detect instance-level performance bottlenecks, and diagnose load balancing issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12975, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log gc skipped", "group": "Raft Log", "used_for": "Used for monitoring the rate of skipped Raft log garbage collection operations in TiKV, broken down by instance and skip reason. Essential for diagnosing issues where Raft logs aren't being cleaned up properly, which can lead to excessive disk usage and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_raft_log_gc_skipped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572555, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log async fetch task duration", "group": "Raft Log", "used_for": "Used for monitoring the latency of asynchronous Raft log entry fetch operations in TiKV at p95 and p99 percentiles. Critical for diagnosing replication delays and identifying performance bottlenecks when followers retrieve log entries from leaders during Raft consensus.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftlog-fetch-worker\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 80, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring query execution latency at multiple percentiles (p80, p95, p99, p999) across TiDB instances. Essential for identifying performance degradation, tail latency issues, and understanding query response time distribution when investigating slow query complaints or SLA violations.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 260, "dashboard": "Prometheus-TiCDC-Clinic", "title": "EtcdWorker txn size percentile", "group": "Etcd", "used_for": "Used for monitoring the 95th and 99th percentile transaction size in bytes for TiCDC's etcd worker operations. Essential for identifying abnormally large etcd transactions that could impact TiCDC's metadata management performance and detecting potential issues with change data capture coordination overhead.", "component": "TICDC", "phenomenon": "", "object": "transaction", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 291, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Etcd 99% WAL fsync duration", "group": "Etcd", "used_for": "Used for monitoring the 99th percentile fsync latency of etcd's Write-Ahead Log (WAL) in TiCDC. Critical for diagnosing etcd performance issues and disk I/O bottlenecks that can impact TiCDC's metadata operations and change data capture reliability.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 21, "dashboard": "TiDB-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "Used for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for understanding query workload distribution, identifying traffic patterns, and detecting sudden changes in application behavior or load spikes.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572229, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Raft log fetch", "group": "Raft Log", "used_for": "Used for monitoring the rate of Raft log entry fetches in TiKV, categorized by fetch type. Helps diagnose Raft replication issues, identify excessive log fetching that may indicate follower lag or network problems, and understand the distribution of different fetch operations across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_entry_fetches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 137, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per minute in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing which types of errors (e.g., syntax errors, permission issues, or execution timeouts) are affecting the cluster.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 60, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC CPU", "group": "TiKV", "used_for": "Used for monitoring CPU usage of TiCDC-related threads running on TiKV instances, including CDC workers, CDC event processors, and TSO threads. Essential for diagnosing performance bottlenecks in change data capture operations and identifying when CDC workloads are consuming excessive CPU resources on storage nodes.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[2m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2292, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Local reader requests", "group": "Local reader", "used_for": "Used for monitoring the rate of local read requests in TiKV, including successfully executed reads, stale reads, and rejected requests with their reasons. Essential for diagnosing local reader performance issues and understanding why read requests might be rejected (e.g., due to lease expiration or data staleness).", "component": "TIKV", "phenomenon": "io", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_raftstore_local_read_reject_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)", "sum(rate(tikv_raftstore_local_read_executed_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 42, "dashboard": "TiDB-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (success/error). Essential for tracking overall query throughput, comparing current performance against historical baselines (1-day offset), and quickly identifying sudden drops in successful queries or spikes in failed commands that indicate cluster health issues.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 267, "dashboard": "TiDB-Cluster-Clinic", "title": "Affected Rows By Type", "group": "Query Summary", "used_for": "Used for monitoring the rate of rows affected by different SQL statement types (INSERT, UPDATE, DELETE, etc.) in TiDB. Helps identify data modification patterns, detect unusual write activity, and diagnose issues related to bulk operations or unexpected data changes.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (sql_type)", "sum(rate(tidb_server_affected_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 74, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC network traffic", "group": "TiKV", "used_for": "Used for monitoring the network traffic sent by TiKV's CDC (Change Data Capture) gRPC messages, broken down by instance and message type. Essential for diagnosing CDC replication bottlenecks, network bandwidth issues, and identifying which TiKV instances are generating the most CDC traffic.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4196, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of read operations hitting Level 0 SST files in TiKV's unified read pool. High Level 0 chance indicates potential read amplification and compaction pressure, which can degrade read performance and signal the need for tuning RocksDB compaction settings.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4194, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiKV's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate workload imbalances or priority queue bottlenecks affecting read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 134, "dashboard": "TiDB-Cluster-Clinic", "title": "99 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution latency in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation and diagnosing which query types are experiencing elevated latency that affects the majority of users.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 147, "dashboard": "Prometheus-TiCDC-Clinic", "title": "gRPC message count", "group": "TiKV", "used_for": "Used for monitoring the rate of gRPC message requests handled by TiKV instances, broken down by message type. Essential for identifying which gRPC operations are most active and diagnosing communication bottlenecks between TiCDC and TiKV during data replication.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 136, "dashboard": "TiDB-Cluster-Clinic", "title": "999 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile (P999) query execution duration in TiDB, broken down by SQL statement type. Critical for identifying tail latency issues and detecting slow query outliers that affect a small percentage of requests but can significantly impact user experience.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC memory", "group": "TiKV", "used_for": "Used for monitoring memory consumption of TiKV instances and CDC (Change Data Capture) components. Helps diagnose memory-related issues in TiKV nodes, particularly when troubleshooting CDC replication performance problems or investigating potential memory leaks in the capture process.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"tikv.*\"}) by (instance)", "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"cdc.*\"}) by (instance)", "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 153, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved Region", "group": "TiKV", "used_for": "Used for monitoring the minimum resolved timestamp region across TiKV instances in TiCDC replication. Helps identify lagging regions that may be blocking CDC checkpoint advancement and causing replication delays.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 132, "dashboard": "TiDB-Cluster-Clinic", "title": "95 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration in TiDB, broken down by SQL statement type (SELECT, INSERT, UPDATE, etc.). Essential for identifying performance degradation affecting most users and detecting when 95% of queries complete within acceptable latency thresholds.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572469, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Unified Read Pool Wait Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the wait time distribution of tasks in TiKV's unified read pool before execution begins. Critical for diagnosing read request queuing issues and identifying when the read pool is saturated, causing increased latency for coprocessor and storage read operations.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unified-read.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 112, "dashboard": "TiDB-Cluster-Clinic", "title": "Slow query", "group": "Query Summary", "used_for": "Used for monitoring the 90th percentile latency breakdown of slow queries in TiDB, including total processing time, coprocessor execution time, and wait time. Essential for diagnosing performance bottlenecks in slow queries and identifying whether delays occur during query processing, storage layer operations, or resource waiting.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))", "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_mvcc_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le,sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4198, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiKV's unified read pool across instances. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient thread pool capacity or slow read operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 152, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Min resolved ts", "group": "TiKV", "used_for": "Used for monitoring the replication lag between TiKV's minimum resolved timestamp and the current cluster TSO in TiCDC data capture scenarios. Critical for identifying CDC replication delays and ensuring change data capture keeps pace with upstream transactions.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 218, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Idle Duration", "group": "Query Summary", "used_for": "Used for monitoring the idle time duration of database connections in TiDB, separated by transaction state (in_txn='1' for active transactions, in_txn='0' for idle connections). Essential for identifying connection pool inefficiencies, detecting long-running idle transactions that may hold locks, and optimizing application connection management patterns.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='1'}[2m])) by (le,in_txn))", "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", in_txn='0'}[2m])) by (le,in_txn))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 70, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Resolved ts lag duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.999th percentile lag between TiKV's resolved timestamp and current time in TiCDC replication. Critical for identifying extreme tail latency in change data capture, helping diagnose replication delays and ensuring data consistency in downstream systems.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 80 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution duration across individual TiDB instances. Helps identify performance variations between instances and pinpoint which specific TiDB nodes are experiencing slower query processing times.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 4204, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Schedule Times", "group": "Unified Read Pool", "used_for": "**Used For:**\n\nUsed for monitoring task execution latency at various percentiles (p50, p95, p99, p999) in TiKV's unified read pool. Essential for diagnosing read performance issues and identifying scheduling delays that may impact query response times, particularly when investigating slow reads or I/O bottlenecks in the storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_execute_times_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 72, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan duration percentile", "group": "TiKV", "used_for": "Used for monitoring the 99.99th percentile latency of TiCDC's initial table scan operations in TiKV. Critical for diagnosing slow initial replication setup and identifying outlier scan performance issues that could delay change data capture initialization.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 130, "dashboard": "TiDB-Cluster-Clinic", "title": "80 Duration", "group": "Query Summary", "used_for": "**Used For:**\n\nUsed for monitoring the 80th percentile query execution latency in TiDB, showing the duration threshold that 80% of queries complete within. Helps identify typical query performance and detect degradation affecting the majority of queries, making it useful for understanding baseline performance and catching early signs of slowdowns before they impact tail latencies.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 80 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le,sql_type))", "histogram_quantile(0.80, tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 143, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache hit", "group": "TiKV", "used_for": "Used for monitoring the cache hit rate of TiCDC's old value cache in TiKV instances. Essential for evaluating TiCDC replication efficiency and identifying memory pressure issues that could impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)", "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4199, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Duration of One Time Slice", "group": "Unified Read Pool", "used_for": "**Used For:**\n\nUsed for monitoring the duration of individual task execution time slices in TiKV's unified read pool at various percentiles (p50, p95, p99, p999). Essential for diagnosing read request processing efficiency and identifying whether tasks are being preempted too frequently or holding execution threads too long, which can impact overall read performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_poll_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 78, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Captured region count", "group": "TiKV", "used_for": "Used for monitoring the number of regions captured by TiCDC on each TiKV instance and their resolution status. Essential for diagnosing TiCDC replication issues, identifying regions that are stuck or slow to replicate, and ensuring change data capture is working correctly across all regions.", "component": "TIKV", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4202, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Task Execute Duration", "group": "Unified Read Pool", "used_for": "Used for monitoring the execution duration of tasks in TiKV's unified read pool at various percentiles (p50, p95, p99, p999). Essential for diagnosing read performance issues and identifying whether the unified read pool is experiencing task execution delays that could impact query latency.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 50, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring the rate of different storage commands (Get, Scan, Prewrite, Commit, etc.) processed by TiKV instances. Essential for understanding workload patterns, identifying command-specific bottlenecks, and diagnosing transaction processing issues across the cluster.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 68, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV takes to perform initial data scans during TiCDC changefeed initialization. Critical for diagnosing slow changefeed startup issues and understanding the overhead of historical data scanning when creating new CDC tasks.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 15, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous snapshot requests in TiKV's storage engine. Critical for diagnosing performance issues with read operations that require consistent snapshots, such as point queries and range scans.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 25, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 99 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99th percentile query execution duration for each TiDB instance. Helps identify performance bottlenecks and uneven load distribution across individual TiDB servers by comparing p99 latencies between instances.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.99, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC scan speed", "group": "TiKV", "used_for": "Used for monitoring the rate at which TiKV scans data for CDC (Change Data Capture) replication, measured in bytes per second per instance. Essential for diagnosing CDC performance bottlenecks, identifying slow TiKV nodes during initial table scans, and ensuring efficient data synchronization to downstream systems.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 139, "dashboard": "Prometheus-TiCDC-Clinic", "title": "CDC total scan bytes", "group": "TiKV", "used_for": "Used for monitoring the total bytes scanned by TiCDC (Change Data Capture) on each TiKV instance. Helps identify data replication workload intensity and diagnose performance issues when TiCDC is reading historical data or catching up on change logs.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", component=\"tikv\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 140, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Initial scan tasks status", "group": "TiKV", "used_for": "Used for monitoring the status of TiCDC initial scan tasks in TiKV, showing both ongoing and completed scans per instance. Essential for tracking TiCDC replication initialization progress and identifying instances with stalled or slow initial data synchronization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 8, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the rate of failed asynchronous storage requests in TiKV, broken down by error status type. Critical for identifying storage engine issues, detecting data access failures, and diagnosing underlying problems with RocksDB operations or disk I/O errors.", "component": "TIKV", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 141, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek operation", "group": "TiKV", "used_for": "Used for monitoring the rate of old value scan operations in TiKV's Change Data Capture (CDC) feature, broken down by instance, column family, and tag. Essential for diagnosing CDC performance issues and understanding the I/O overhead of scanning historical data when capturing data changes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (instance, cf, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 146, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the time TiKV spends seeking old values during CDC (Change Data Capture) operations. Critical for diagnosing CDC replication performance issues and identifying bottlenecks in historical data retrieval for change feeds.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 20000, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiKV's storage engine at p95 and p99 percentiles. Critical for diagnosing read performance issues and identifying delays in obtaining consistent snapshots for transactions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 109, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying when async write operations are experiencing delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 31112, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "Used for monitoring the time spent waiting for read index proposals in TiKV's storage engine. Critical for diagnosing read latency issues caused by Raft consensus delays, particularly when follower reads experience slow proposal acknowledgments from the leader.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 94, "dashboard": "TiDB-Cluster-Clinic", "title": "Failed Query OPM Detail", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the per-minute increase in query execution errors across TiDB instances. Essential for identifying which specific TiDB servers are experiencing query failures and diagnosing application-level errors, connection issues, or SQL syntax problems in real-time.", "component": "TIDB", "phenomenon": "error_rate", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 68, "dashboard": "TiDB-Cluster-Clinic", "title": "Internal SQL OPS", "group": "Query Detail", "used_for": "Used for monitoring the throughput of internal SQL operations executed by TiDB, such as statistics collection, auto-analyze, and system maintenance queries. Essential for distinguishing internal database overhead from user workload and diagnosing performance issues caused by background tasks.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_restricted_sql_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 81, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 999 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 99.9th percentile query execution duration for each individual TiDB instance. Essential for identifying specific TiDB nodes experiencing tail latency issues or performance degradation, enabling targeted troubleshooting of slow query problems at the instance level.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.999, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 20001, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the p95 and p99 latency of asynchronous write operations in TiKV's storage engine. Critical for diagnosing write performance bottlenecks and identifying when storage layer write requests are experiencing delays that could impact transaction commit times.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 272, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries in Multi-Statement", "group": "Query Detail", "used_for": "Used for monitoring the average number of queries contained within multi-statement requests and the total rate of queries in multi-statement batches. Helps identify applications using batch query patterns and diagnose potential performance issues related to multi-statement execution overhead.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))/sum(rate(tidb_server_multi_query_num_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tidb_server_multi_query_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3, "dashboard": "TiDB-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring TiDB server memory consumption, including process resident memory and Go heap allocation. Essential for detecting memory leaks, identifying out-of-memory risks, and optimizing memory configuration to prevent TiDB instance crashes.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_free_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_gc_heap_goal_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}", "sum(tidb_server_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb\"}) by (module, instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 321, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Receive Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages received by TiCDC instances from peer nodes. Helps identify communication bottlenecks, network issues, or abnormal message traffic patterns between TiCDC nodes during data replication.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31111, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency distribution of asynchronous snapshot operations that perform pure local reads in TiKV's storage engine. Critical for diagnosing performance issues with local read operations and identifying when snapshot acquisition is causing delays in query processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 184, "dashboard": "TiDB-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Essential for tracking server stability, identifying recent restarts or crashes, and verifying cluster availability during troubleshooting or maintenance operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 142, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value seek duration", "group": "TiKV", "used_for": "Used for monitoring the latency of TiKV seeking old values during TiCDC change data capture operations, showing p95 and p99 percentiles. Critical for diagnosing TiCDC replication performance issues and identifying instances where historical value lookups are causing delays in change feed processing.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag))", "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[2m])) by (le, instance, tag)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 323, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Send Rate", "group": "Peer Messages", "used_for": "Used for monitoring the rate of messages sent by TiCDC instances to downstream systems or other peers. Helps identify message throughput bottlenecks, replication lag issues, and abnormal communication patterns in the change data capture pipeline.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "ticdc_instance" ], "promql": [ "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[2m]))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 168, "dashboard": "TiDB-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring CPU utilization of TiDB server instances compared to their configured CPU limits (maxprocs). Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiDB instances need vertical scaling or if query workloads need optimization.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])", "tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8, "dashboard": "TiDB-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances and tracking connection token usage. Essential for diagnosing connection exhaustion issues, identifying connection leaks, and determining if the cluster is approaching connection limits that could cause new connection failures.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})", "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 188, "dashboard": "TiDB-Cluster-Clinic", "title": "Open FD Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of open file descriptors in TiDB server processes. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause service disruptions.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 31113, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's storage engine. Critical for diagnosing read performance issues related to Raft read index protocol, especially when investigating slow snapshot reads or follower read delays.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration 95 By Instance", "group": "Query Detail", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile query execution duration for each TiDB instance. Helps identify performance variations across individual TiDB nodes and pinpoint which specific instances are experiencing elevated query latency.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))", "histogram_quantile(0.95, tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate{instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 352, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Message Batch Size", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message batch sizes in TiCDC's peer-to-peer communication at p50, p95, and p99 percentiles. Helps diagnose message batching efficiency and identify potential network or replication bottlenecks when batch sizes are unexpectedly small or large.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 61, "dashboard": "TiDB-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiDB instances. Essential for detecting goroutine leaks, resource exhaustion issues, and identifying abnormal concurrency patterns that could indicate connection pooling problems or stuck operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_sched_goroutines_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=~\"tidb.*\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 31115, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index propose wait duration", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring the wait duration (p95, p99, and average) for read index propose operations in TiKV's storage engine. Critical for diagnosing read latency issues caused by delays in proposing read index requests to the Raft layer, particularly when investigating slow queries or read performance degradation in strongly consistent read scenarios.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_propose_wait\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 350, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Stream Count Between Nodes", "group": "Peer Messages", "used_for": "**Used For:**\n\nUsed for monitoring the current number of active message streams between TiCDC nodes in the cluster. Helps diagnose inter-node communication issues, connection stability problems, and potential network bottlenecks affecting data replication between TiCDC instances.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "ticdc_message_server_cur_stream_count" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31116, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Read index confirm duration", "group": "Storage", "used_for": "Used for monitoring the latency of read index confirmation operations in TiKV's storage engine, showing p95 and p99 percentiles. Critical for diagnosing read performance issues related to Raft read index verification, which ensures linearizable reads in distributed transactions.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_read_index_confirm\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 356, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive Message Bytes Percentile", "group": "Peer Messages", "used_for": "Used for monitoring the distribution of message sizes received by TiCDC's message server at different percentiles (p50, p95, p99). Helps identify abnormally large messages in peer-to-peer communication and diagnose network bandwidth issues or inefficient data transfer patterns between TiCDC nodes.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 31114, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Storage async snapshot duration (pure local read)", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations that read data purely from local TiKV storage (p95 and p99 percentiles). Critical for diagnosing local read performance issues and identifying when snapshot acquisition delays are impacting query response times in scenarios without remote data fetching.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_local_read\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 354, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Receive message batch bytes percentiles", "group": "Peer Messages", "used_for": "Used for monitoring the size distribution of message batches received by TiCDC's message server, tracking p50, p95, and p99 percentiles. Essential for diagnosing network efficiency and identifying abnormally large message batches that could indicate replication bottlenecks or memory pressure in peer-to-peer communication.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [], "promql": [ "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 165, "dashboard": "TiDB-Cluster-Clinic", "title": "Prepare Statement Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of prepared statements currently cached in TiDB instances. Helps identify potential memory pressure from excessive prepared statement accumulation and diagnose connection pooling or statement lifecycle management issues in applications.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_prepared_stmts{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 54, "dashboard": "TiDB-Cluster-Clinic", "title": "Panic And Critial Error", "group": "Server", "used_for": "Used for monitoring panic events and critical errors occurring in TiDB server instances. Essential for immediately detecting severe system failures and stability issues that require urgent investigation and intervention.", "component": "TIDB", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_panic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", "increase(tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 205, "dashboard": "TiDB-Cluster-Clinic", "title": "Disconnection Count", "group": "Server", "used_for": "Used for tracking the total number of client disconnections from TiDB instances, categorized by disconnection result type. Essential for diagnosing connection stability issues, identifying abnormal client disconnection patterns, and troubleshooting network or client-side problems that may impact application availability.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 11512, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Throttle duration", "group": "Flow Control", "used_for": "Used for monitoring the cumulative time TiKV scheduler operations spend in throttled state due to flow control mechanisms. Essential for diagnosing performance degradation caused by resource pressure, helping identify when write operations are being intentionally delayed to prevent system overload.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_scheduler_throttle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9552, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler flow", "group": "Flow Control", "used_for": "Used for monitoring write flow and throttling activity in TiKV's scheduler across instances. Helps identify when flow control is actively throttling writes due to pressure, indicating potential write bottlenecks or resource constraints that may impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_write_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_throttle_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9553, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler discard ratio", "group": "Flow Control", "used_for": "Used for monitoring the ratio of discarded scheduler commands in TiKV instances due to flow control mechanisms. Helps identify when TiKV is rejecting requests because of I/O pressure or resource constraints, indicating potential performance bottlenecks or overload conditions.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_discard_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 11906, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler throttled CF", "group": "Flow Control", "used_for": "Used for monitoring which column families (CFs) in TiKV are being throttled by the scheduler due to flow control mechanisms. Helps identify write pressure and backpressure situations where specific column families are experiencing resource constraints or reaching their processing limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tikv_scheduler_throttle_cf{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} != 0" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 49, "dashboard": "TiDB-Cluster-Clinic", "title": "Events OPM", "group": "Server", "used_for": "Used for monitoring the rate of server events occurring in TiDB instances, measured as operations per minute over a 10-minute window. Helps identify abnormal event patterns, server state changes, or potential issues by tracking the frequency of internal server events like connection changes, schema updates, or other significant operational activities.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "increase(tidb_server_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10023, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo fsync duration", "group": "Redo", "used_for": "Used for monitoring the fsync operation latency in TiCDC's redo log system. Critical for diagnosing redo log write performance issues and identifying disk I/O bottlenecks that could impact data replication reliability and changefeed lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 82, "dashboard": "TiDB-Cluster-Clinic", "title": "Keep Alive OPM", "group": "Server", "used_for": "Used for monitoring the rate of keep-alive operations per minute sent by TiDB instances to maintain connections with clients or other components. Helps diagnose connection stability issues, detect abnormal connection churn, or identify instances experiencing connectivity problems.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_keep_alive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10025, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flushall duration", "group": "Redo", "used_for": "Used for monitoring the duration of redo log flush-all operations in TiCDC, which writes accumulated redo logs to persistent storage. Critical for diagnosing redo log write performance issues and identifying bottlenecks in disaster recovery scenarios where redo logs ensure data consistency.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 145, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Old value cache size", "group": "TiKV", "used_for": "Used for monitoring TiCDC's old value cache in TiKV, tracking the number of cached entries, total memory usage, and memory quota per instance. Critical for diagnosing TiCDC replication issues and ensuring the old value cache doesn't exceed memory limits, which could impact change data capture performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster", "tikv_instance" ], "promql": [ "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 358, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Total Stream Count", "group": "Peer Messages", "used_for": "Used for monitoring the total number of active peer message streams in TiCDC and comparing it against the expected stream count based on running TiCDC instances. Helps identify stream connection issues, network problems, or TiCDC node communication failures when actual streams deviate from the expected count (2N-1 for N nodes).", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(ticdc_message_server_cur_stream_count)", "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"ticdc\"}) - 1" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 111, "dashboard": "TiDB-Cluster-Clinic", "title": "Get Token Duration", "group": "Server", "used_for": "Used for monitoring the 99th percentile latency of token acquisition in TiDB's connection rate limiting mechanism. Critical for diagnosing connection throttling issues and identifying when clients experience delays due to token bucket rate limiting during high concurrency scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 191, "dashboard": "TiDB-Cluster-Clinic", "title": "Skip Binlog Count", "group": "Server", "used_for": "Used for monitoring the count of critical errors related to binlog skipping in TiDB instances. Essential for identifying binlog replication issues and data consistency problems when TiDB fails to write binlog entries, which could impact downstream replication to TiDB Binlog or other CDC tools.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 166, "dashboard": "TiDB-Cluster-Clinic", "title": "Time Jump Back OPS", "group": "Server", "used_for": "Used for detecting system clock backward jumps on TiDB instances, measured as operations per second. Critical for identifying time synchronization issues that can cause transaction timestamp anomalies and data consistency problems in distributed transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_monitor_time_jump_back_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9947, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller actions", "group": "Flow Control", "used_for": "Used for monitoring the rate of flow control actions taken by TiKV's scheduler to prevent write overload. Essential for diagnosing write throttling issues and understanding when TiKV is actively limiting write operations to protect system stability under heavy I/O pressure.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_throttle_action_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cf, type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10031, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write rows/s ", "group": "Redo", "used_for": "Used for monitoring the rate of rows written to TiCDC's redo log per second, broken down by instance and changefeed. Essential for tracking redo log write throughput and identifying performance bottlenecks or anomalies in the change data capture pipeline's persistence layer.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)", "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[2m])) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 211, "dashboard": "TiDB-Cluster-Clinic", "title": "Client Data Traffic", "group": "Server", "used_for": "Used for monitoring the network traffic volume between TiDB servers and clients, broken down by traffic type (inbound/outbound). Essential for identifying network bottlenecks, unusual traffic patterns, and diagnosing connectivity issues or data transfer anomalies.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10029, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo flush log duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log flush operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery feature when redo logs are being persisted to storage, helping identify slow disk I/O or storage system issues affecting replication reliability.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10033, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo Write bytes/s ", "group": "Redo", "used_for": "Used for monitoring the redo log write throughput (bytes per second) for each TiCDC instance. Essential for tracking redo log I/O performance and identifying instances with high write volume or potential I/O bottlenecks during changefeed replication.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 167, "dashboard": "TiDB-Cluster-Clinic", "title": "Handshake Error OPS", "group": "Server", "used_for": "Used for monitoring the rate of client connection handshake failures per TiDB instance. Essential for diagnosing authentication issues, SSL/TLS configuration problems, or client compatibility issues that prevent successful database connections.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_handshake_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 9944, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flow controller factors", "group": "Flow Control", "used_for": "Used for monitoring the key factors that trigger TiKV's write flow control mechanism, including L0 SST file count, memtable count, and average L0 files. Essential for diagnosing write throttling issues and understanding when TiKV applies backpressure to prevent storage engine overload.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_scheduler_l0{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_memtable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_l0_avg{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10338, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Flush/L0 flow", "group": "Flow Control", "used_for": "**Used For:**\n\nUsed for monitoring the flow control metrics related to L0 SST files and memtable flush operations in TiKV's RocksDB storage engine. Essential for diagnosing write stalls and understanding when flow control throttling is triggered due to excessive L0 files or pending flush operations, which can impact write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_l0_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_flush_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 9946, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "Flow Control", "used_for": "Used for monitoring the volume of data waiting to be compacted in TiKV's RocksDB storage engine, broken down by column family. Critical for diagnosing write stalls and flow control issues, as excessive pending compaction bytes indicate the storage engine cannot keep up with write pressure, potentially triggering write throttling to prevent system overload.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"}) by (cf)", "max(tikv_scheduler_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) / 10000000" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10027, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Redo write log duration", "group": "Redo", "used_for": "Used for monitoring the latency distribution of redo log write operations in TiCDC. Critical for diagnosing performance bottlenecks in TiCDC's disaster recovery feature when redo logs are being persisted to storage.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])) by (le)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 23763572363, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time transaction commands spend being throttled due to flow control mechanisms in TiKV. Essential for diagnosing performance degradation caused by resource pressure, helping identify when transactions are being delayed by backpressure from storage layer congestion or quota limits.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_txn_command_throttle_time_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763572365, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Non-txn command throttled duration", "group": "Flow Control", "used_for": "Used for monitoring the time spent throttling non-transactional commands (like raw KV operations) due to flow control in TiKV. Essential for diagnosing performance degradation when non-txn operations are being rate-limited, helping identify whether flow control is impacting client request latency.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "sum(rate(tikv_non_txn_command_throttle_time_total{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 158, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Estimated Live Objects", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring the number of live objects in TiCDC's Go runtime heap memory. Helps diagnose memory leaks and excessive object allocation issues that could lead to increased garbage collection pressure and performance degradation.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 171, "dashboard": "TiDB-Cluster-Clinic", "title": "RCCheckTS WriteConflict Num", "group": "Server", "used_for": "Used for monitoring the number of write conflicts detected during read-committed isolation level timestamp checks in TiDB. Helps diagnose transaction contention issues and identify workloads experiencing high conflict rates that may impact write performance.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_rc_check_ts_conflict_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 160, "dashboard": "Prometheus-TiCDC-Clinic", "title": "GC STW Duration (last 256 GC cycles)", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring Go garbage collection stop-the-world (STW) pause durations in TiCDC instances over the last 256 GC cycles. Critical for diagnosing replication latency issues caused by excessive GC pauses that can block TiCDC's change data capture processing.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 157, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Memory Usage", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory consumption, including resident memory, heap allocation, and garbage collection thresholds. Essential for detecting memory leaks, identifying OOM risks, and optimizing TiCDC's memory configuration during change data capture operations.", "component": "TICDC", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 168, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the total number of scheduler operations by stage and tracking \"too busy\" rejections for a specific command type in TiKV. Helps diagnose scheduler bottlenecks, resource contention, and identify which processing stages are experiencing high load or rejecting requests due to capacity limits.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 161, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Allocator Throughput", "group": "Runtime $runtime_instance", "used_for": "Used for monitoring TiCDC's memory allocation throughput, including allocation rate, deallocation rate, and object allocation frequency. Critical for diagnosing memory pressure issues, garbage collection overhead, and identifying memory leaks or excessive allocation patterns in TiCDC instances.", "component": "TICDC", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "runtime_instance", "tidb_cluster" ], "promql": [ "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])", "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[2m])" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 3, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the P95 and P99 latency of TiKV scheduler commands (like prewrite, commit, acquire_pessimistic_lock). Essential for diagnosing transaction processing performance issues and identifying slow scheduler operations that may impact overall database responsiveness.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10035, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Worker Busy Ratio", "group": "Redo", "used_for": "Used for monitoring the busy ratio of TiCDC redo log workers, indicating how much time workers spend actively processing versus idle. Essential for identifying redo log processing bottlenecks and determining if workers are saturated, which could lead to replication lag or require scaling adjustments.", "component": "TICDC", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "changefeed", "k8s_cluster", "ticdc_instance", "tidb_cluster" ], "promql": [ "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[2m])/10) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 72, "dashboard": "TiDB-Cluster-Clinic", "title": "Duration", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile transaction duration in TiDB, broken down by transaction type (optimistic/pessimistic) and showing both percentile and average latencies. Essential for identifying transaction performance degradation, comparing transaction modes, and diagnosing slow commit issues that impact application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, txn_mode))", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode) / sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", scope=~\"general\"}[2m])) by (txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 195, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiKV, showing p95, p99, and average metrics. Essential for diagnosing I/O efficiency issues and identifying commands that scan excessive keys, which may indicate inefficient queries or hotspot problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10002, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication of DDL schema changes to Pulsar topics in TiCDC. Essential for verifying that schema evolution events are properly propagated to downstream Pulsar consumers and diagnosing DDL replication issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10001, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of DDL schema changes published to Pulsar topics by TiCDC. Essential for tracking schema synchronization activity and diagnosing issues when DDL operations aren't being replicated correctly to downstream Pulsar systems.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 69, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring transaction operations per second in TiDB, broken down by transaction type (commit/rollback) and mode (optimistic/pessimistic). Essential for tracking transaction throughput, identifying workload patterns, and diagnosing issues when transaction rates drop unexpectedly or differ significantly between general user transactions and internal system transactions.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (type, txn_mode)", "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"internal\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 74, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Statement Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of statement counts per transaction in TiDB. Helps identify transactions with excessive statements that may cause performance issues, long-running transactions, or inefficient batch operations requiring optimization.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 67, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction retry attempts in TiDB. Helps diagnose transaction conflicts, write contention issues, and identify workloads experiencing frequent optimistic lock failures that may benefit from pessimistic locking or application-level optimization.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_retry_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 36, "dashboard": "TiDB-Cluster-Clinic", "title": "Session Retry Error OPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate of transaction retry errors in TiDB sessions, categorized by error type and SQL type. Essential for diagnosing transaction conflicts, lock contention issues, and identifying which types of queries are experiencing retry failures that could impact application performance.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_retry_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, sql_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 194, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time transactions spend waiting to acquire memory locks (latches) in TiKV's scheduler before executing commands. Critical for diagnosing transaction contention issues and identifying hotspot keys that cause concurrent transactions to queue, especially when p99 latency is elevated.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 560, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's scheduler, broken down by operation tags. Helps diagnose scan-heavy workloads and identify which types of scan operations (e.g., range scans, index lookups) are consuming scheduler resources.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10003, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DDL schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring DDL schema publication failures when TiCDC replicates data to Pulsar. Critical for diagnosing issues where DDL changes fail to be published to Pulsar topics, which can cause replication inconsistencies or changefeed failures.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 4, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction OPS", "group": "Transaction", "used_for": "Used for monitoring the throughput of KV transaction operations in TiDB, broken down by operation type (commit, rollback, etc.) per instance. Essential for identifying transaction bottlenecks, analyzing workload patterns, and diagnosing issues when transaction processing capacity becomes saturated.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (instance, type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 196, "dashboard": "TiDB-Cluster-Clinic", "title": "Commit Token Wait Duration", "group": "Transaction", "used_for": "Used for monitoring the wait time transactions spend acquiring commit tokens in TiDB's batch executor at p80, p95, and p99 percentiles. Critical for diagnosing transaction commit bottlenecks caused by token contention when concurrent commits exceed available token capacity, which can indicate the need to adjust `committer-concurrency` settings.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 373, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiKV, showing p95, p99, and average metrics. Essential for identifying write-heavy transactions, diagnosing scheduler bottlenecks, and detecting abnormal batch write patterns that may impact cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 675, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations specifically on the lock column family during scheduler command execution in TiKV. Essential for diagnosing transaction conflicts and lock contention issues by tracking how frequently the scheduler scans lock-related data, helping identify bottlenecks in transaction processing.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 44, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Regions Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of region counts involved in transaction prewrite operations. Helps identify transactions that span many regions, which can indicate inefficient data access patterns or potential performance bottlenecks in distributed transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572710, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler command read duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the P95 and P99 latency of read operations in TiKV's scheduler for specific command types. Essential for diagnosing slow read performance issues and identifying whether the scheduler's read processing is causing query latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 193, "dashboard": "TiDB-Cluster-Clinic", "title": "KV Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution of KV transaction operations in TiDB at the 80th, 95th, and 99th percentiles. Essential for diagnosing transaction performance issues, identifying slow transaction types (commit, prewrite, etc.), and detecting latency degradation that impacts application response times.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 829, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations on TiKV's write column family during scheduler command execution. Helps diagnose write-heavy workload patterns and identify potential performance bottlenecks related to scanning operations in the write path.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10005, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema success", "group": "Pulsar Sink", "used_for": "Used for monitoring the successful publication rate of DML schema changes to Pulsar topics in TiCDC. Helps verify that schema changes are being correctly propagated to downstream Pulsar systems and identify potential schema synchronization issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 201, "dashboard": "TiDB-Cluster-Clinic", "title": "Statement Lock Keys", "group": "Transaction", "used_for": "Used for monitoring the distribution of lock keys acquired per statement in TiDB transactions. Helps diagnose transaction contention issues and identify statements that acquire excessive locks, which can lead to performance degradation and deadlocks.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 7236, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Check memory locks duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the time spent checking memory locks during transaction processing in TiKV's scheduler. Critical for diagnosing transaction conflicts and lock contention issues that can cause increased latency in write-heavy workloads.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_storage_check_mem_lock_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10004, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema count", "group": "Pulsar Sink", "used_for": "Used for monitoring the rate of DML schema changes published to Pulsar topics by TiCDC. Helps track schema evolution frequency and identify changefeeds experiencing high schema change activity, which could impact downstream consumers or indicate potential replication issues.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 209, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num", "group": "Transaction", "used_for": "Used for monitoring the distribution of key-value pairs written per transaction in TiDB. Helps identify transactions with excessive write operations that may cause performance bottlenecks or hotspot issues in the cluster.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10008, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client connections opened", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of active Pulsar client connections opened by TiCDC changefeeds. Helps diagnose connection management issues, detect connection leaks, and verify proper connectivity between TiCDC and Pulsar message brokers during data replication.", "component": "TICDC", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 830, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiKV's default column family, broken down by operation tags. Helps diagnose scan performance issues and identify which types of scan operations (e.g., user scans, GC scans) are consuming resources in the scheduler.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10006, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar published DML schema fail", "group": "Pulsar Sink", "used_for": "Used for monitoring failed DML schema publications to Pulsar topics in TiCDC. Critical for identifying schema synchronization issues when replicating data changes to Pulsar, helping diagnose why certain table schemas fail to publish and may cause replication delays or data inconsistencies.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 1, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiKV. Helps identify workload distribution across different priority queues and diagnose issues where high-priority commands may be blocked by lower-priority operations.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 33, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write KV Num Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative count of key-value pairs written per transaction in TiDB. Helps identify transaction write patterns, detect abnormal batch sizes, and diagnose issues related to large transactions that may impact cluster performance or cause write hotspots.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_kv_num_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10007, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client bytes published ", "group": "Pulsar Sink", "used_for": "Used for monitoring the total bytes published by TiCDC's Pulsar client to the Pulsar message queue, broken down by changefeed and instance. Essential for tracking data replication throughput to Pulsar and diagnosing potential bottlenecks or network issues in the CDC pipeline.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10009, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client rpc count", "group": "Pulsar Sink", "used_for": "Used for monitoring the total number of RPC calls made by TiCDC's Pulsar client to the Pulsar cluster, broken down by changefeed and instance. Helps diagnose communication issues between TiCDC and Pulsar, identify abnormal RPC patterns, and troubleshoot data replication problems in Pulsar sink scenarios.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 194, "dashboard": "TiDB-Cluster-Clinic", "title": "Send HeartBeat Duration", "group": "Transaction", "used_for": "Used for monitoring the latency of transaction heartbeat messages sent from TiDB to TiKV to keep pessimistic locks alive. Critical for diagnosing transaction timeout issues and lock maintenance problems in long-running transactions, especially when locks are held beyond the default TTL.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 34, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes Rate and Sum", "group": "Transaction", "used_for": "Used for monitoring the rate and cumulative volume of data written by TiDB transactions to TiKV. Essential for analyzing transaction write throughput patterns, identifying write-heavy workloads, and diagnosing performance issues related to large transaction sizes or sudden increases in write volume.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}[2m])", "tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", scope=~\"general\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 23763572468, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Txn Scheduler Pool Wait Duration", "group": "Scheduler", "used_for": "Used for monitoring the wait time distribution of transactions in TiKV's scheduler worker pool before being processed. Critical for diagnosing transaction processing bottlenecks and identifying when the scheduler is overloaded, causing transactions to queue excessively before execution.", "component": "TIKV", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched-worker.*\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 210, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Write Size Bytes", "group": "Transaction", "used_for": "Used for monitoring the distribution of transaction write sizes in TiDB, showing how many bytes are written per transaction. Helps identify large transactions that may cause performance issues, memory pressure, or exceed transaction size limits, and is essential for diagnosing write-heavy workload patterns.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\", scope=~\"general\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 3834, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "**Used For:**\n\nUsed for monitoring the volume of data being written by TiKV's scheduler component on each instance. Helps identify write-heavy instances, diagnose unbalanced write distribution across the cluster, and detect potential write bottlenecks or abnormal write traffic patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 224, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Commit .99 Backoff", "group": "Transaction", "used_for": "Used for monitoring the 99th percentile backoff duration during transaction commits in TiDB. Critical for diagnosing transaction commit delays caused by lock conflicts, region unavailability, or other transient issues that trigger retry mechanisms.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 193, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands queued in each TiKV instance. Helps identify scheduler bottlenecks and command processing delays that could indicate storage layer congestion or resource saturation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 197, "dashboard": "TiDB-Cluster-Clinic", "title": "Acquire Pessimistic Locks Duration", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the latency of acquiring pessimistic locks in TiDB transactions at p80, p95, and p99 percentiles. Critical for diagnosing transaction performance issues, lock contention problems, and identifying when pessimistic locking operations are causing delays in concurrent workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10011, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer rpc latency", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile RPC latency of Pulsar client producers in TiCDC's Pulsar sink. Critical for identifying tail latency issues when TiCDC replicates change data to Pulsar, helping diagnose network delays or Pulsar broker performance problems that could cause replication lag.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 167, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiKV, including both normal operations and \"too busy\" rejections. Essential for diagnosing scheduler bottlenecks, identifying when TiKV is overloaded and rejecting requests, and understanding the distribution of work across scheduler stages like snapshot, apply, and write.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 195, "dashboard": "TiDB-Cluster-Clinic", "title": "TTL Lifetime Reach Counter", "group": "Transaction", "used_for": "Used for monitoring the rate at which transaction TTL (Time To Live) lifetimes are reached in TiDB. Helps identify transactions that are holding locks for too long, which can indicate potential lock contention issues, slow transactions, or applications not properly committing/rolling back transactions.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 83, "dashboard": "TiDB-Cluster-Clinic", "title": "Load Safepoint OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of successful safepoint loading operations in TiDB's transaction system. Helps diagnose GC (Garbage Collection) coordination issues and transaction snapshot management problems, as safepoints are critical for maintaining MVCC consistency across the cluster.", "component": "TIDB", "phenomenon": "throughput", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_load_safepoint_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ok\"}[2m])) by (instance)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 199, "dashboard": "TiDB-Cluster-Clinic", "title": "Pessimistic Statement Retry OPS", "group": "Transaction", "used_for": "Used for monitoring the rate of pessimistic transaction statement retries in TiDB. Helps identify transaction conflicts and contention issues that cause statements to retry, which can indicate lock conflicts or write-write conflicts in pessimistic transaction mode.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 10010, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer latency ", "group": "Pulsar Sink", "used_for": "Used for monitoring the 99.9th percentile latency of TiCDC's Pulsar client when producing messages to Pulsar topics. Critical for identifying tail latency issues in the data replication pipeline from TiDB to Pulsar, helping diagnose slow message delivery and potential bottlenecks in the sink component.", "component": "TICDC", "phenomenon": "latency", "object": "", "percentile": [ 999 ], "variables": [ "changefeed" ], "promql": [ "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[2m])) by (le,changefeed,instance))\r", "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[2m])) by (changefeed,instance)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 10012, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in TiCDC's Pulsar producer queue that are waiting to be sent to Pulsar. Critical for identifying message backlog issues, producer bottlenecks, or downstream Pulsar cluster performance problems that could lead to replication lag.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed" ], "promql": [ "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 2224, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks duration", "group": "GC", "used_for": "Used for monitoring the P95, P99, and maximum duration of garbage collection tasks in TiKV. Essential for diagnosing GC performance issues that could impact cluster stability, identifying slow GC operations, and ensuring old MVCC versions are being cleaned up efficiently without causing latency spikes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, task))", "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 10013, "dashboard": "Prometheus-TiCDC-Clinic", "title": "Pulsar client producer pending messages", "group": "Pulsar Sink", "used_for": "Used for monitoring the number of pending messages in the Pulsar client producer queue for TiCDC changefeeds. Helps identify message backlog issues and potential bottlenecks in data replication to Pulsar, indicating when the producer cannot keep up with the change data capture rate.", "component": "TICDC", "phenomenon": "", "object": "", "percentile": [], "variables": [ "changefeed", "schema", "topic" ], "promql": [ "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)" ], "dashboard_file": "prometheus-ticdc.json" }, { "panel_id": 121, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC tasks", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection tasks in TiKV, including successful GC operations, skipped tasks, and failed tasks by type. Essential for diagnosing GC performance issues, identifying task failures, and understanding why certain GC operations are being skipped, which can impact storage space reclamation and MVCC version cleanup.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_tasks_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_storage_gc_skipped_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_gcworker_gc_task_fail_vec{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (task)", "sum(rate(tikv_gc_worker_too_busy{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 966, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC worker actions", "group": "GC", "used_for": "Used for monitoring the rate of different garbage collection (GC) worker actions in TiDB, broken down by action type. Essential for diagnosing GC performance issues, identifying bottlenecks in the cleanup process, and understanding which GC operations are consuming the most resources.", "component": "TIDB", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_gc_worker_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 226, "dashboard": "TiDB-Cluster-Clinic", "title": "SafeTS Update Conuter", "group": "Transaction", "used_for": "Used for monitoring the rate of SafeTS (Safe Timestamp) updates between TiDB and TiKV stores, categorized by update result and store. Essential for diagnosing transaction coordination issues, timestamp synchronization problems, and identifying stores with abnormal SafeTS update patterns that could impact transaction consistency.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_safets_update_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result, store)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 969, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiDB GC seconds", "group": "GC", "used_for": "Used for monitoring the maximum (100th percentile) duration of garbage collection operations initiated by TiDB to clean up obsolete MVCC versions in TiKV. Critical for diagnosing GC performance issues that could lead to storage bloat, increased read latency, or cluster instability when GC operations take too long.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance, le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2823, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "ResolveLocks Progress", "group": "GC", "used_for": "Used for monitoring the progress and results of lock resolution operations in TiKV during garbage collection. Helps diagnose issues with transaction cleanup and identify stuck or failed lock resolution tasks that could block GC progress.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_range_task_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=~\"resolve-locks.*\"}) by (result)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 252, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the P99, P90, and P80 latency of different transaction execution states in TiDB. Essential for diagnosing transaction performance bottlenecks by identifying which specific transaction phases (idle, executing, committing, etc.) are experiencing high latency or delays.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))", "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 212, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction Types Per Second", "group": "Transaction", "used_for": "Used for monitoring the rate of different transaction commit types per second in TiDB, including standard two-phase commit, async commit, and one-phase commit (1PC) transactions. Essential for understanding transaction workload patterns and identifying which commit protocols are being used, helping diagnose transaction performance issues and validate optimization strategies like async commit adoption.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_tikvclient_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_async_commit_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_tikvclient_one_pc_txn_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 8767, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan write details", "group": "GC", "used_for": "Used for monitoring the rate of keys scanned during garbage collection in TiKV's write column family, broken down by key mode and tag. Essential for diagnosing GC performance issues, understanding GC workload patterns, and identifying potential bottlenecks in the write CF cleanup process.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\", cf=\"write\"}[2m])) by (key_mode,tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 225, "dashboard": "TiDB-Cluster-Clinic", "title": "Max SafeTS Gap", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the maximum gap between SafeTS (Safe Timestamp) and current time in TiDB's transaction system. Critical for diagnosing stale read issues, transaction consistency problems, and detecting when TiKV nodes fall behind in timestamp synchronization, which can impact read performance and data freshness.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_tikvclient_min_safets_gap_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 2821, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC Progress", "group": "GC", "used_for": "Used for monitoring the progress of TiKV's automatic garbage collection process by tracking the ratio of scanned regions to total regions per instance. Helps operators verify GC is progressing normally across the cluster and identify instances where GC may be stuck or lagging behind.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_gcworker_autogc_processed_regions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2822, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "TiKV Auto GC SafePoint", "group": "GC", "used_for": "Used for monitoring the automatic garbage collection (GC) safe point timestamp in TiKV, which indicates the oldest data version that must be retained. Essential for diagnosing GC progress issues, MVCC version accumulation problems, and ensuring transactions can access historical data within the configured retention period.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tikv_gcworker_autogc_safe_point{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (instance) / (2^18)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 8768, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC scan default details", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection key scanning operations in TiKV's default column family, broken down by key mode and tag. Helps diagnose GC performance issues and understand the volume of keys being processed during garbage collection cycles.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gcworker_gc_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",instance=~\"$instance\", cf=\"default\"}[2m])) by (key_mode,tag)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 2589, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC speed", "group": "GC", "used_for": "Used for monitoring the rate at which TiKV's garbage collection (GC) deletes obsolete MVCC versions. Essential for verifying GC efficiency and diagnosing issues when storage space isn't being reclaimed as expected or when GC is impacting cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 255, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction execution states duration", "group": "Transaction", "used_for": "Used for monitoring the time spent in different transaction execution states and overall transaction duration in TiDB. Helps diagnose transaction performance bottlenecks by identifying which states (e.g., lock waiting, commit, rollback) are consuming the most time and whether transactions are experiencing delays.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_txn_state_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)", "sum(rate(tidb_session_transaction_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 295, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Usage", "group": "Transaction", "used_for": "Used for monitoring the usage patterns of TiDB's fair locking mechanism in transactions, categorized by lock type. Helps diagnose transaction contention issues and understand how frequently fair locking is being triggered to prevent lock starvation in high-concurrency scenarios.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_session_transaction_fair_locking_usage{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 253, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction with lock execution states duration", "group": "Transaction", "used_for": "Used for monitoring the latency distribution (p80, p90, p99) of transaction execution states that hold locks in TiDB. Critical for diagnosing lock contention issues and identifying transactions that may be blocking others, helping pinpoint the root cause of transaction performance degradation and deadlock scenarios.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 90, 99 ], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\", has_lock=\"true\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 254, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction enter state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions enter different states (e.g., idle, running, committed) in TiDB. Helps diagnose transaction lifecycle issues, identify abnormal state transitions, and understand transaction processing patterns during performance troubleshooting.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 261, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction state count change", "group": "Transaction", "used_for": "Used for monitoring the rate of change in transaction state transitions within TiDB sessions. Helps identify transaction state management issues, such as transactions getting stuck in certain states or abnormal state transition patterns that could indicate deadlocks or connection problems.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 231, "dashboard": "TiDB-Cluster-Clinic", "title": "Assertion", "group": "Transaction", "used_for": "Used for monitoring the rate of prewrite assertion operations in TiDB transactions, categorized by assertion type. Helps diagnose transaction conflicts and optimistic locking issues that can impact transaction latency and throughput.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 6596, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC in Compaction Filter", "group": "GC", "used_for": "Used for monitoring the rate of garbage collection operations performed during RocksDB compaction in TiKV. Helps diagnose GC efficiency by tracking filtered keys, skipped operations, and MVCC rollback events, which is essential for understanding storage cleanup performance and identifying potential GC bottlenecks during compaction.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_gc_compaction_filtered{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_skip{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_mvcc_rollback{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_orphan_versions{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_perform{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_failure{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode,type)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_met{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_handled{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_wasted{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (key_mode)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 37, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Snapshot generation wait duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile wait time before snapshot generation begins in TiKV's Raft store. Critical for diagnosing delays in snapshot creation that can impact region replication, leader transfers, and new replica bootstrapping performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 256, "dashboard": "TiDB-Cluster-Clinic", "title": "Transaction leave state", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring the rate at which transactions exit different states in TiDB, categorized by transaction type. Helps diagnose transaction lifecycle issues, identify abnormal transaction patterns, and troubleshoot problems related to transaction commit, rollback, or timeout behaviors.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_session_txn_state_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 28, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC interval", "group": "GC", "used_for": "Used for monitoring the configured interval between garbage collection (GC) runs in TiKV. Helps verify GC scheduling configuration and diagnose issues when old MVCC versions aren't being cleaned up as expected, which can lead to storage bloat and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"tikv_gc_run_interval\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 35, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent between TiKV instances. Essential for diagnosing cluster rebalancing activity, replica recovery operations, and identifying potential network or storage bottlenecks during large data transfers.", "component": "TIKV", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 156, "dashboard": "TiDB-Cluster-Clinic", "title": "Parse Duration", "group": "Executor", "used_for": "**Used For:**\n\nUsed for monitoring the 95th percentile SQL parsing latency in TiDB, broken down by SQL statement type. Essential for identifying parsing bottlenecks when queries take unexpectedly long to execute, particularly with complex SQL statements or when experiencing high query compilation overhead.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 38, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots by type (sending/receiving/applying) in TiKV. Essential for diagnosing region replication issues, identifying snapshot transfer bottlenecks, and understanding cluster data synchronization patterns during node recovery or scaling operations.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 154, "dashboard": "TiDB-Cluster-Clinic", "title": "Compile Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile SQL compilation latency in TiDB, broken down by SQL statement type. Critical for diagnosing slow query plan generation issues and identifying when the query optimizer is taking excessive time to compile SQL statements before execution.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 76, "dashboard": "TiDB-Cluster-Clinic", "title": "Expensive Executors OPS", "group": "Executor", "used_for": "Used for monitoring the rate of expensive executor operations in TiDB, categorized by executor type. Helps identify query patterns that consume excessive resources (memory, CPU) and may impact cluster performance or cause OOM issues.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_expensive_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 258, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Miss OPS", "group": "Executor", "used_for": "Used for monitoring the rate of plan cache misses per second in TiDB, categorized by miss type. Essential for diagnosing SQL execution performance issues caused by inefficient plan cache utilization, which can lead to increased query compilation overhead and reduced throughput.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 169, "dashboard": "TiDB-Cluster-Clinic", "title": "Execution Duration", "group": "Executor", "used_for": "Used for monitoring the 95th percentile execution duration of SQL statements in TiDB, broken down by SQL type (SELECT, INSERT, UPDATE, etc.). Essential for identifying slow query execution patterns and diagnosing performance degradation in specific SQL workload types.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, sql_type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 91, "dashboard": "TiDB-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache feature, broken down by cache type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 296, "dashboard": "TiDB-Cluster-Clinic", "title": "Fair Locking Keys", "group": "Transaction", "used_for": "Used for monitoring the rate of fair locking operations in TiDB's pessimistic transaction model, categorized by locking type. Helps diagnose lock contention issues and identify when transactions are aggressively acquiring locks, which can indicate hotspot problems or inefficient transaction patterns affecting cluster performance.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance" ], "promql": [ "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 44, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of snapshot sizes in TiKV, representing the largest snapshots being transferred during region replication or recovery. Critical for diagnosing extreme cases of snapshot transfer overhead that could impact cluster performance and network bandwidth.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 23763573704, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiKV, including sending, applying, and generating snapshots during Raft replication. Critical for diagnosing slow region recovery, replica synchronization delays, and identifying performance bottlenecks when TiKV nodes handle snapshot transfers for data rebalancing or new replica bootstrapping.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 43, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pairs accessed per snapshot operation in TiKV. Critical for identifying extreme cases where transactions read an unusually large number of keys, which can indicate inefficient queries or potential performance bottlenecks in snapshot isolation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 4201, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot transport speed", "group": "Snapshot", "used_for": "Used for monitoring the rate of snapshot data transfer and generation in TiKV, measured in bytes per second. Essential for diagnosing snapshot replication performance issues, identifying bandwidth bottlenecks during region migration, and understanding the speed of data synchronization between TiKV nodes.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "tidb_cluster" ], "promql": [ "rate(tikv_snapshot_limit_transport_bytes{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])", "rate(tikv_snapshot_limit_generate_bytes{tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 249, "dashboard": "TiDB-Cluster-Clinic", "title": "Read From Table Cache OPS", "group": "Executor", "used_for": "Used for monitoring the rate of read operations served directly from TiDB's table cache feature. Helps assess table cache effectiveness and identify whether cached small tables are reducing storage layer access for frequently queried data.", "component": "TIDB", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_read_from_tablecache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 269, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Memory Usage", "group": "Executor", "used_for": "Used for monitoring the memory consumption of TiDB's plan cache on each instance. Essential for diagnosing memory pressure issues and determining if plan cache memory usage is contributing to OOM problems or needs tuning.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 271, "dashboard": "TiDB-Cluster-Clinic", "title": "Plan Cache Plan Num", "group": "Executor", "used_for": "Used for monitoring the total number of cached execution plans stored in TiDB's plan cache per instance. Helps assess plan cache memory usage and effectiveness, useful when diagnosing performance issues related to plan cache capacity or investigating whether frequently executed queries are being properly cached.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_plan_cache_instance_plan_num_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 59, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiKV, broken down by worker name. Essential for identifying worker thread bottlenecks, uneven task distribution, and diagnosing performance issues related to specific background operations like apply, scheduler, or raftstore workers.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 36, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Snapshot Actions", "group": "Snapshot", "used_for": "Used for monitoring Raft snapshot operations in TiKV, including snapshot generation, application, and region cleanup activities. Essential for diagnosing region replication issues, identifying snapshot transfer bottlenecks, and troubleshooting scenarios involving region splits, merges, or peer recovery.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tikv_raftstore_snapshot_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, status)", "sum(delta(tikv_raftstore_clean_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1395, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the number of pending tasks queued in TiKV's internal worker threads. Essential for identifying thread pool saturation, task backlogs, and potential bottlenecks in background operations like compaction, scheduling, or apply workers that could impact overall cluster performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1877, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiKV's FuturePool thread pools. Helps identify thread pool saturation and task queuing issues that can lead to request processing delays and performance degradation.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 3062, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "Request duration", "group": "Coprocessor Overview", "used_for": "Used for monitoring the distribution of coprocessor request latency in TiKV across all percentiles. Essential for diagnosing slow coprocessor operations like table scans and index lookups that impact query performance.", "component": "TIKV", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 1876, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by TiKV's FuturePool thread pools, broken down by pool name. Helps identify thread pool utilization patterns and diagnose performance bottlenecks when specific pools become saturated or show abnormal task processing rates.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 27, "dashboard": "TiKV-Details-Cluster-Clinic", "title": "GC lifetime", "group": "GC", "used_for": "Used for monitoring the configured GC (Garbage Collection) lifetime setting in TiKV, which determines how long historical MVCC versions are retained. Essential for diagnosing issues related to data retention policies, storage space usage, and ensuring the GC lifetime is appropriately configured to balance between historical data access needs and storage efficiency.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tidb_tikvclient_gc_config{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"tikv_gc_life_time\"})" ], "dashboard_file": "tikv_detail.json" }, { "panel_id": 312, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Latency", "group": "Executor", "used_for": "Used for monitoring the latency of MPP (Massively Parallel Processing) coordinator operations in TiDB at different percentiles (p50, p75, p95). Essential for diagnosing performance issues in analytical queries that use TiFlash, helping identify coordination overhead and bottlenecks in distributed query execution across MPP workers.", "component": "TIDB", "phenomenon": "latency", "object": "", "percentile": [ 50, 75, 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.75, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_executor_mpp_coordinator_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 311, "dashboard": "TiDB-Cluster-Clinic", "title": "Mpp Coordinator Counter", "group": "Executor", "used_for": "Used for monitoring the rate of MPP (Massively Parallel Processing) coordinator operations in TiDB, broken down by operation type. Essential for diagnosing MPP query execution issues and understanding the coordinator's workload when troubleshooting distributed analytical query performance.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_mpp_coordinator_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 12, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql Duration", "group": "Distsql", "used_for": "Used for monitoring the latency distribution of distributed SQL (DistSQL) query execution in TiDB at various percentiles (p50, p90, p99, p999). Essential for diagnosing performance issues when TiDB pushes query operations down to TiKV, helping identify slow distributed query processing and coprocessor request bottlenecks.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 50, 90, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 14, "dashboard": "TiDB-Cluster-Clinic", "title": "Distsql QPS", "group": "Distsql", "used_for": "Used for monitoring the distributed SQL query throughput in TiDB, broken down by coprocessor request type (table scan, index scan, etc.). Essential for identifying which types of distributed queries are consuming the most resources and diagnosing performance bottlenecks in query execution across TiKV nodes.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_distsql_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (copr_type)" ], "dashboard_file": "tidb_detail.json" }, { "panel_id": 32, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of applying Raft logs in TiFlash-Proxy per server instance. Critical for diagnosing replication lag and identifying TiFlash nodes experiencing slow Raft log application, which can impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 118, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Process ready duration per server", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile duration of Raft ready state processing in TiFlash-Proxy per server. Critical for diagnosing Raft replication delays and identifying performance bottlenecks in TiFlash's consensus layer that could impact data synchronization and query freshness.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='ready'}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 5, "dashboard": "Lightning-Cluster-Clinic", "title": "Idle workers", "group": "", "used_for": "Used for monitoring the number of idle worker threads in TiDB Lightning during data import operations. Helps identify resource utilization issues - too many idle workers may indicate bottlenecks in data processing or suboptimal parallelism configuration.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_idle_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 228, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Status Change OPS", "group": "Pipeline Model", "used_for": "Used for monitoring the rate of pipeline task status transitions in TiFlash's pipeline execution model. Helps diagnose task scheduling efficiency and identify potential bottlenecks when tasks frequently change states, indicating possible resource contention or execution issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_pipeline_task_change_to_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\r\n" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 18, "dashboard": "Lightning-Cluster-Clinic", "title": "Deliver duration", "group": "", "used_for": "Used for monitoring the latency of data delivery operations during TiDB Lightning import processes. Critical for diagnosing bottlenecks in the data ingestion pipeline, including range delivery to TiKV, SST file delivery, and row-based KV pair delivery from Lightning to the cluster.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_row_kv_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_kv_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 17, "dashboard": "Lightning-Cluster-Clinic", "title": "Total bytes", "group": "", "used_for": "Used for monitoring data volume metrics during TiDB Lightning import operations, including bytes read from source data, bytes delivered to TiKV, and total cluster storage size. Essential for tracking import progress, estimating completion time, and ensuring sufficient storage capacity during large-scale data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_row_read_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}", "pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"} / ignoring(type) pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"max_replicas\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 77, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Handling Request Number", "group": "Coprocessor", "used_for": "Used for monitoring the number of coprocessor requests currently being processed by TiFlash instances, categorized by request type. Helps identify request processing bottlenecks and diagnose issues when TiFlash is handling high concurrent workloads or experiencing request queuing.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4, "dashboard": "Lightning-Cluster-Clinic", "title": "Import Progress", "group": "", "used_for": "Used for monitoring the progress of data import operations in TiDB Lightning by tracking the ratio of finished chunks to estimated total chunks. Essential for estimating import completion time and identifying stalled or slow import jobs that may require intervention.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"} / ignoring(state) lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"estimated\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 7, "dashboard": "Lightning-Cluster-Clinic", "title": "Memory usage", "group": "", "used_for": "Used for monitoring memory consumption of TiDB Lightning and its importer component during data import operations. Critical for detecting memory leaks, preventing OOM issues, and ensuring sufficient resources are allocated for large-scale data migration tasks.", "component": "", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}", "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 16, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process rate", "group": "", "used_for": "Used for monitoring the throughput rate of SQL data delivery during TiDB Lightning import operations. Helps identify data ingestion bottlenecks and track the overall progress of bulk data loading into the cluster.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "lightning.json" }, { "panel_id": 100, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the number of active RPC threads in TiFlash instances handling coprocessor requests. Essential for diagnosing thread pool saturation, identifying instances under heavy RPC load, and troubleshooting coprocessor request processing bottlenecks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 14, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Response Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the rate of response data bytes per second returned by TiFlash coprocessor requests, broken down by request type. Helps identify data transfer bottlenecks and diagnose network bandwidth issues when TiFlash returns large result sets to TiDB.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 267, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Scheduler", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's segment scheduler threads, which manage data segment operations like merging and splitting. Essential for diagnosing performance bottlenecks in TiFlash's storage layer and identifying when segment management tasks are consuming excessive CPU resources.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3688, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log commit operations in TiFlash-Proxy, broken down by server instance. Critical for diagnosing write path performance issues and identifying slow commit operations that could impact data replication and consistency in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 5, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ready handled", "group": "Raft process", "used_for": "Used for monitoring the rate of Raft ready operations handled by TiFlash-Proxy, broken down by operation type. Essential for diagnosing Raft consensus processing performance and identifying bottlenecks in TiFlash's replication pipeline when data synchronization from TiKV appears slow or stalled.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ready\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1615, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Sent messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages sent by each TiFlash-Proxy instance to other nodes in the cluster. Helps identify communication bottlenecks, network issues, or instances with abnormally high/low message sending rates that could indicate replication problems or cluster imbalance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3690, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Commit log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of committing Raft logs to disk in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing write path performance issues and identifying when disk I/O bottlenecks are causing replication delays in TiFlash's Raft storage layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1215, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD heartbeats", "group": "PD", "used_for": "Used for monitoring the rate of heartbeat messages exchanged between TiFlash-Proxy and PD (Placement Driver). Essential for diagnosing cluster health issues, detecting communication failures, and identifying when TiFlash nodes lose connectivity with PD.", "component": "PD", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 153, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Sender", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's snapshot sender threads, which handle sending Raft snapshots to other replicas. Critical for diagnosing performance issues during region replication, data migration, or when new TiFlash nodes join the cluster.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1396, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD validate peers", "group": "PD", "used_for": "Used for monitoring the rate of peer validation operations between TiFlash-Proxy and PD (Placement Driver). Helps diagnose issues with region peer metadata synchronization and identify problems in the communication between TiFlash-Proxy and PD during peer validation processes.", "component": "PD", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 19, "dashboard": "Lightning-Cluster-Clinic", "title": "SST process duration", "group": "", "used_for": "Used for monitoring the performance of SST (Sorted String Table) file processing during TiDB Lightning data import operations. Helps diagnose import bottlenecks by tracking SST chunk throughput, split duration, and upload latency to TiKV nodes.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"split\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"write\"}[2m])\n", "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"ingest\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 246, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "TiFlash Resource Group", "group": "TiFlash Resource Control", "used_for": "Used for monitoring TiFlash resource group consumption and throttling behavior, tracking remaining tokens, average consumption speed, and total resource usage per group. Essential for diagnosing resource control effectiveness, identifying resource-constrained groups, and ensuring fair resource allocation across different workloads in TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"remaining_tokens\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"avg_speed\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"total_consumption\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_fill_rate\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_capacity\", instance=~\"$instance\"}) by (instance,resource_group)", "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fetch_tokens_from_gac_count\", instance=~\"$instance\"}) by (instance,resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 161, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Client", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async client threads that handle remote procedure calls. Essential for diagnosing communication bottlenecks between TiFlash and other TiDB components, especially when experiencing high network request latency or thread saturation issues.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 8, "dashboard": "Lightning-Cluster-Clinic", "title": "Failures", "group": "", "used_for": "Used for monitoring the number of table import failures in TiDB Lightning across the cluster. Critical for quickly identifying data migration issues and tracking which tables failed to import successfully.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", result=\"failure\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 13, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk parser read block duration", "group": "", "used_for": "Used for monitoring the average duration of chunk parser read block operations and I/O apply worker latency in TiDB Lightning during data import. Critical for diagnosing data ingestion performance bottlenecks and identifying slow file reading or parsing issues that may impact overall import speed.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_chunk_parser_read_block_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_chunk_parser_read_block_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_apply_worker_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) /rate(lightning_apply_worker_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name = \"io\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 670, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region task worker pre-handle/generate snapshot CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's region task and worker threads responsible for pre-handling region operations and generating snapshots. Critical for diagnosing TiFlash region replication bottlenecks and identifying instances with high CPU consumption during snapshot generation or region data synchronization.", "component": "TIFLASH", "phenomenon": "throughput", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_task.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"region_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 40, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration per server", "group": "Raft IO", "used_for": "Used for monitoring the 99th percentile latency of Raft log append operations in TiFlash-Proxy per server instance. Critical for diagnosing write performance bottlenecks and identifying slow disk I/O issues in TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 63, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Cop task memory usage", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring memory consumption of coprocessor requests in TiFlash at various percentiles (p80-p999). Essential for identifying memory-intensive queries, detecting potential OOM risks, and optimizing TiFlash resource allocation when coprocessor tasks consume excessive memory.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 156, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Large Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background threads handling large tasks like compaction and data reorganization. Essential for diagnosing performance bottlenecks when background operations consume excessive CPU resources, impacting query performance or causing resource contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3, "dashboard": "Lightning-Cluster-Clinic", "title": "CPU%", "group": "", "used_for": "Used for monitoring CPU utilization percentage of TiDB Lightning and Importer processes during data import operations. Essential for identifying resource bottlenecks, ensuring adequate CPU allocation, and diagnosing performance degradation during bulk data loading tasks.", "component": "", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}[2m])*100", "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"importer\"}[2m])*100" ], "dashboard_file": "lightning.json" }, { "panel_id": 15, "dashboard": "Lightning-Cluster-Clinic", "title": "SQL process duration", "group": "", "used_for": "Used for monitoring the latency of SQL processing stages during Lightning data import, specifically tracking row encoding duration and block delivery duration. Essential for diagnosing performance bottlenecks in the Lightning import pipeline and identifying whether slowdowns occur during data encoding or delivery phases.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(lightning_row_encode_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_row_encode_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 123, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "0.99 Duration of raft store events", "group": "Raft process", "used_for": "Used for monitoring the 99th percentile latency of Raft store events in TiFlash-Proxy, tracking how long different Raft operations take to complete. Critical for diagnosing TiFlash replication lag and identifying performance bottlenecks in Raft log processing, particularly when investigating slow data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "store", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "Lightning-Cluster-Clinic", "title": "Number of Lightning Goroutines", "group": "", "used_for": "Used for monitoring the number of active goroutines in TiDB Lightning instances during data import operations. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact import performance or cause Lightning to hang.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tidb-lightning\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 149, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Manual Compaction", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's manual compaction thread pool. Helps identify when manual compaction operations are consuming excessive CPU resources, which may indicate heavy background maintenance activity or potential performance bottlenecks in storage layer optimization.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 218, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of CPU Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's CPU task thread pool, which handles compute-intensive operations in the pipeline execution model. Essential for identifying CPU bottlenecks, thread pool saturation, and optimizing TiFlash query performance under heavy analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Error QPS", "group": "Coprocessor", "used_for": "Used for monitoring the rate of coprocessor request errors per second in TiFlash, broken down by error reason. Essential for identifying and diagnosing issues with query execution failures, helping pinpoint whether errors stem from timeout, region unavailability, or other coprocessor-related problems.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1616, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Flush messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft message flushes per TiFlash-Proxy server instance. Helps identify message processing bottlenecks and uneven Raft communication load distribution across TiFlash nodes, which can indicate network issues or imbalanced cluster workload.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 13, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Handle Duration", "group": "Coprocessor", "used_for": "Used for monitoring the latency distribution (p80, p95, p99, p999) of coprocessor request handling in TiFlash. Essential for diagnosing slow analytical queries and identifying performance degradation in TiFlash's request processing pipeline, particularly when users experience delayed query responses or need to investigate tail latency issues in OLAP workloads.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 151, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Storage Background (Small Tasks)", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash storage background worker threads that handle small maintenance tasks like compaction and cleanup. Helps identify resource contention or performance bottlenecks in background operations that could impact overall TiFlash storage efficiency.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10, "dashboard": "Lightning-Cluster-Clinic", "title": "Chunk process duration", "group": "", "used_for": "Used for monitoring the average time taken to process each data chunk during TiDB Lightning import operations. Essential for identifying import performance bottlenecks and estimating overall data loading completion time.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "1/rate(lightning_chunks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"finished\"}[2m]) " ], "dashboard_file": "lightning.json" }, { "panel_id": 9, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second (QPS) of coprocessor requests handled by TiFlash, broken down by request type. Essential for tracking TiFlash's analytical query workload and identifying sudden changes in request patterns or throughput bottlenecks.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "Lightning-Cluster-Clinic", "title": "Checksum progress", "group": "", "used_for": "Used for tracking the completion progress of table checksum verification during TiDB Lightning data import. Helps monitor how many tables have completed checksum validation versus the total, essential for ensuring data integrity and estimating remaining import time.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"completed\"} / ignoring(state) lightning_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"pending\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 104, "dashboard": "DM-Professional-Clinic", "title": "Task State", "group": "", "used_for": "Used for monitoring the current state of DM (Data Migration) worker tasks across different sources and jobs. Essential for tracking task health, identifying stuck or failed migration tasks, and ensuring data replication is running smoothly.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 220, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of IO Task Thread Pool", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's IO task thread pool, which handles asynchronous I/O operations. Essential for identifying CPU bottlenecks in I/O processing and determining if the IO thread pool is saturated or underutilized during heavy read/write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 268, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Segment Reader", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SegmentReader threads, which handle reading data from storage segments during query execution. Essential for identifying compute bottlenecks in TiFlash's columnar storage layer and diagnosing slow analytical query performance caused by excessive segment reading overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 230, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Max Execute Time Per Round", "group": "Pipeline Model", "used_for": "Used for monitoring the maximum execution time of pipeline tasks per scheduling round in TiFlash at p95 and p99 percentiles. Critical for identifying pipeline scheduling inefficiencies, task execution imbalances, and potential bottlenecks in TiFlash's query processing pipeline that could impact overall query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 6, "dashboard": "Lightning-Cluster-Clinic", "title": "External resources", "group": "", "used_for": "Used for monitoring the number of active external resources (KV encoders, importer engines, and open engine RPC calls) during Lightning data import operations. Helps identify resource leaks or accumulation issues when resources are opened but not properly closed, which can lead to memory exhaustion or import failures.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_kv_encoder{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"open\"} - ignoring(type) lightning_importer_engine{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"closed\"}", "tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"open_engine\",result=\"ok\"} - ignoring(request) tikv_import_rpc_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", request=\"close_engine\",result=\"ok\"}" ], "dashboard_file": "lightning.json" }, { "panel_id": 31, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of applying Raft logs in TiFlash-Proxy's storage engine, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication lag and identifying bottlenecks in the Raft log application process that could impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 216, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads CPU of Wait Reactor", "group": "Pipeline Model", "used_for": "Used for monitoring CPU utilization of TiFlash's Wait Reactor threads, which handle asynchronous I/O operations and event processing in the pipeline execution model. Essential for diagnosing whether wait reactors are becoming CPU bottlenecks during query execution or if thread pool sizing needs adjustment.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 226, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Count", "group": "Pipeline Model", "used_for": "Used for monitoring the number of pipeline tasks in TiFlash's execution engine, showing both per-instance and cluster-wide task counts. Essential for diagnosing TiFlash query execution bottlenecks, identifying task accumulation issues, and understanding workload distribution across instances.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (instance, type)", "sum(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 11, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Messages", "group": "Raft message", "used_for": "Used for monitoring the rate of different types of Raft messages sent by TiFlash-Proxy nodes in the cluster. Helps diagnose Raft communication patterns and identify potential issues with consensus protocol message flow, such as excessive heartbeats, vote requests, or append entries that could indicate network problems or cluster instability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 222, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Duration", "group": "Pipeline Model", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of pipeline tasks in TiFlash's pipeline execution model. Essential for identifying performance bottlenecks in TiFlash query processing and diagnosing slow task execution issues that impact overall query latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r", "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m]))\r\n\r\n\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m]))\r\n\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m]))\r", "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m]))\r" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1069, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD requests", "group": "PD", "used_for": "Used for monitoring the rate of different types of PD (Placement Driver) requests sent from TiFlash-Proxy instances. Helps identify communication patterns between TiFlash-Proxy and PD, and diagnose issues related to metadata operations, region information queries, or TSO (timestamp oracle) requests.", "component": "PD", "phenomenon": "", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1, "dashboard": "Lightning-Cluster-Clinic", "title": "Import speed", "group": "", "used_for": "Used for monitoring the data import throughput during TiDB Lightning operations, tracking write speed to TiKV, upload speed, and overall imported bytes per second. Essential for diagnosing import performance bottlenecks and ensuring data migration tasks are proceeding at expected rates.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])", "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"imported\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"restored\"}[2m])", "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", state=\"written\"}[2m])" ], "dashboard_file": "lightning.json" }, { "panel_id": 21, "dashboard": "Lightning-Cluster-Clinic", "title": "Import/Normal mode", "group": "", "used_for": "Used for monitoring TiKV's RocksDB hard pending compaction bytes limit configuration across instances during Lightning import operations. Helps identify whether TiKV is in import mode (higher limit) or normal mode (lower limit), which is critical for diagnosing write performance issues and understanding cluster configuration state during bulk data imports.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "min(tikv_config_rocksdb{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"hard_pending_compaction_bytes_limit\"}) by (instance)" ], "dashboard_file": "lightning.json" }, { "panel_id": 165, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Exchange Bytes/Seconds", "group": "Coprocessor", "used_for": "Used for monitoring the data exchange rate (bytes per second) between TiFlash nodes during distributed query execution, broken down by exchange type. Essential for diagnosing network bottlenecks and data shuffle performance issues in MPP queries.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_exchange_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Executor QPS", "group": "Coprocessor", "used_for": "Used for monitoring the queries per second handled by different TiFlash coprocessor executors, broken down by executor type (e.g., table scan, aggregation, join). Essential for understanding TiFlash workload distribution and identifying which executor operations are consuming the most resources during query processing.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Receive messages per server", "group": "Raft message", "used_for": "Used for monitoring the rate of Raft messages received by each TiFlash-Proxy server instance. Helps identify communication imbalances, network issues, or instances experiencing abnormal Raft message traffic that could indicate replication problems or cluster coordination issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 159, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "GRPC Async Server", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's gRPC async server threads that handle asynchronous network operations. Critical for identifying whether async I/O processing is becoming a bottleneck and if thread pool resources need adjustment.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1070, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "PD request duration (average)", "group": "PD", "used_for": "Used for monitoring the average latency of different types of requests from TiFlash-Proxy to PD (Placement Driver). Critical for diagnosing communication delays between TiFlash compute nodes and PD, helping identify metadata access bottlenecks or PD service degradation affecting TiFlash operations.", "component": "PD", "phenomenon": "latency", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 188, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Usage", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache utilization when using S3-based disaggregated storage, tracking both DTFile cache and page cache capacity versus actual usage. Essential for diagnosing cache efficiency issues and determining if cache sizing is appropriate for workload performance in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_DTFileCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DTFileCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_PageCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 39, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Append log duration", "group": "Raft IO", "used_for": "Used for monitoring the latency of appending Raft logs to disk in TiFlash-Proxy's Raft store, showing p95, p99, and average durations. Critical for diagnosing TiFlash replication performance issues and identifying disk I/O bottlenecks that could slow down data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 11, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Duration", "group": "Coprocessor", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash coprocessor request latency at multiple percentiles (p80, p95, p99, p999). Essential for identifying slow analytical query processing and diagnosing performance degradation in TiFlash's columnar storage engine when handling pushed-down coprocessor requests from TiDB.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 25, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Vote", "group": "Raft message", "used_for": "**Used For:**\n\nUsed for monitoring the rate of Raft vote messages sent by TiFlash-Proxy instances during leader elections. Essential for diagnosing cluster stability issues, frequent leader changes, or split-brain scenarios that trigger excessive voting activity.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"vote\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1709, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memory", "group": "Cluster", "used_for": "Used for monitoring the resident memory usage of TiFlash-Proxy processes across cluster instances. Essential for tracking memory consumption patterns, detecting memory leaks, and ensuring TiFlash-Proxy operates within acceptable memory limits to prevent OOM issues.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "DM-Professional-Clinic", "title": "dump progress", "group": "dump and load ", "used_for": "Used for monitoring the data dump progress in DM (Data Migration) tasks by showing the ratio of finished rows to estimated total rows. Essential for tracking dump phase completion and estimating remaining time during full data migration from source databases.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dumpling_dump_finished_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"} / dumpling_dump_estimate_total_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1309, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft dropped messages", "group": "Raft message", "used_for": "Used for monitoring the rate of dropped Raft messages in TiFlash-Proxy, categorized by message type. Critical for diagnosing Raft communication issues, network problems, or overload conditions that prevent proper consensus protocol operation between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1708, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CPU", "group": "Cluster", "used_for": "Used for monitoring the CPU utilization rate of TiFlash-Proxy processes across the cluster. Essential for identifying CPU bottlenecks, resource contention, and determining if TiFlash-Proxy nodes are overloaded or require scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 157, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "MPP Query count", "group": "Coprocessor", "used_for": "Used for monitoring the count of MPP (Massively Parallel Processing) queries being executed in TiFlash instances. Helps identify query distribution across TiFlash nodes and diagnose workload imbalances or query processing bottlenecks in analytical workloads.", "component": "TIFLASH", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 101, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads of Rpc", "group": "Coprocessor", "used_for": "Used for monitoring the maximum number of RPC threads in TiFlash instances. Helps identify thread pool saturation and potential bottlenecks in handling coprocessor requests, which can indicate overload conditions or the need for thread pool tuning.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 95, "dashboard": "DM-Professional-Clinic", "title": "checksum progress", "group": "dump and load ", "used_for": "Used for monitoring the checksum verification progress during DM's data migration process. Helps track the ratio of completed table checksums to pending checksums, allowing users to estimate completion time and identify potential bottlenecks in the validation phase after data loading.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"completed\"} / ignoring(state) lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"pending\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1710, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "IO utilization", "group": "Cluster", "used_for": "Used for monitoring disk I/O utilization percentage across TiFlash-Proxy nodes in the cluster. Essential for identifying disk bottlenecks and I/O saturation issues that could impact TiFlash query performance and data synchronization.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(node_disk_io_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 102, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Threads", "group": "Coprocessor", "used_for": "Used for monitoring the current thread count across different thread pools in TiFlash instances, excluding RPC and maximum limit threads. Essential for identifying thread resource exhaustion, detecting thread leaks, and diagnosing I/O bottlenecks in TiFlash's coprocessor operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 87, "dashboard": "DM-Professional-Clinic", "title": "load progress", "group": "dump and load ", "used_for": "Used for monitoring the data loading progress in DM's lightning component during full data migration. Shows the ratio of finished chunks to estimated total chunks, helping users track migration completion percentage and estimate remaining time for the load phase.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"} / ignoring(state) lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"estimated\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 32, "dashboard": "DM-Professional-Clinic", "title": "dump process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's dump process over 30-minute intervals. Critical for identifying dump task failures and data migration interruptions that require immediate attention.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_dumpling_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 199, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Time of the Longest Live MPP Task", "group": "Coprocessor", "used_for": "Used for monitoring the execution time of the longest-running MPP (Massively Parallel Processing) task currently active in TiFlash. Helps identify stuck or abnormally long-running distributed query tasks that may indicate performance bottlenecks or resource contention in the MPP execution engine.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_mpp_task_monitor{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 96, "dashboard": "DM-Professional-Clinic", "title": "Chunk process duration", "group": "dump and load ", "used_for": "Used for monitoring the average time taken to process each data chunk during DM's dump and load operations. Essential for identifying performance bottlenecks in data migration tasks and diagnosing slow chunk processing that could delay overall migration completion.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "1/rate(lightning_chunks{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",state=\"finished\"}[2m])" ], "dashboard_file": "dm.json" }, { "panel_id": 42, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft proposals are processed in TiFlash-Proxy's raftstore. Critical for diagnosing replication delays and identifying bottlenecks in the Raft proposal queue that could impact TiFlash data synchronization performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft proposals per ready", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile of Raft proposals processed per ready batch in TiFlash-Proxy. Helps diagnose Raft replication efficiency and identify instances where proposal batching may be suboptimal, potentially indicating I/O bottlenecks or apply worker saturation.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 107, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Min TSO", "group": "Task Scheduler", "used_for": "Used for monitoring the minimum Timestamp Oracle (TSO) value being processed by TiFlash's task scheduler across different resource groups. Helps identify data freshness issues and diagnose replication lag between TiKV and TiFlash, as lower TSO values indicate older data being processed.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"min_tso\"}) by (instance, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 89, "dashboard": "DM-Professional-Clinic", "title": "External resources", "group": "dump and load ", "used_for": "Used for monitoring the number of active external resources (KV encoders and importer engines) during DM's dump and load operations. Helps identify resource leaks or bottlenecks by tracking the difference between opened and closed resources in the Lightning component.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}", "lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 18, "dashboard": "DM-Professional-Clinic", "title": "Idle workers", "group": "dump and load ", "used_for": "Used for monitoring the number of idle worker threads in DM's lightning component during data dump and load operations. Helps identify resource utilization issues, such as insufficient parallelism (too many idle workers) or potential bottlenecks preventing workers from processing data efficiently.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_idle_workers{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1714, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region", "group": "Cluster", "used_for": "Used for monitoring the total number of Raft regions managed by each TiFlash-Proxy instance. Essential for tracking region distribution across TiFlash nodes and diagnosing data skew or unbalanced region allocation issues that could impact query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 119, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft read proposals (including local reads and read index operations) per TiFlash instance. Helps identify read request distribution across TiFlash nodes and diagnose potential read hotspots or imbalanced read workloads in the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 111, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Queries Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting queries in TiFlash's task scheduler across different resource groups. Essential for identifying query queuing issues, resource contention, and determining if TiFlash is overloaded or experiencing scheduling bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_queries_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_queries_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 33, "dashboard": "DM-Professional-Clinic", "title": "load process exits with error", "group": "dump and load ", "used_for": "Used for monitoring error exits in DM's load process during data migration. Helps identify and diagnose failures in the data loading phase by tracking the rate of abnormal process terminations over 30-minute intervals.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_loader_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 113, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Active and Waiting Tasks Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of active and waiting tasks in TiFlash's task scheduler across different resource groups. Helps identify task queue bottlenecks, resource contention, and scheduling inefficiencies when tasks accumulate in waiting state or when active task counts approach capacity limits.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_tasks_count\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_tasks_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4106, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Uptime", "group": "Cluster", "used_for": "Used for monitoring the uptime duration of TiFlash-Proxy instances since their last restart. Helps identify recent restarts, instance stability issues, and verify successful deployments or rolling updates.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "(time() - tiflash_proxy_process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2741, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Critical error", "group": "Errors", "used_for": "Used for monitoring the rate of critical errors in TiFlash-Proxy instances, categorized by error type. Essential for detecting severe operational issues that require immediate attention, such as data corruption, storage failures, or system-level problems that could impact TiFlash availability and data integrity.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_critical_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 166, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data size in send and receive queue", "group": "Coprocessor", "used_for": "Used for monitoring the amount of data queued in TiFlash's exchange operators during coprocessor request processing. Helps identify data transfer bottlenecks and memory pressure when TiFlash nodes exchange intermediate results during distributed query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_exchange_queueing_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1719, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler error", "group": "Errors", "used_for": "Used for monitoring error rates in TiFlash-Proxy's scheduler operations, specifically tracking snapshot and prepare write failures. Critical for diagnosing data replication issues between TiKV and TiFlash, helping identify when the scheduler fails to process write requests or snapshot operations.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[2m])) by (instance, stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server report failures", "group": "Errors", "used_for": "Used for monitoring the rate of server report failures in TiFlash-Proxy, broken down by failure type, instance, and store ID. Critical for identifying communication issues between TiFlash nodes and detecting potential cluster instability or network problems that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type,instance,store_id)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 103, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Max Threads", "group": "Coprocessor", "used_for": "Used for monitoring the maximum thread count limits configured in TiFlash instances across different thread pools. Helps identify thread pool capacity constraints and potential resource bottlenecks when actual thread usage approaches these maximum values.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft Entry Cache", "group": "Cluster", "used_for": "Used for monitoring memory usage of the Raft entry cache in TiFlash-Proxy's raftstore. Helps identify memory pressure issues and potential memory leaks in the Raft log replication layer, which is critical for diagnosing TiFlash replication performance and stability problems.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_server_mem_trace_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore-.*\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 120, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft write proposals per server", "group": "Raft propose", "used_for": "Used for monitoring the rate of Raft write proposals submitted per TiFlash node. Helps identify uneven write distribution across TiFlash instances and diagnose replication bottlenecks or performance issues in the Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"normal\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 117, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Hard Limit Exceeded Count", "group": "Task Scheduler", "used_for": "Used for monitoring the number of times TiFlash task scheduler has exceeded its hard resource limits per instance and resource group. Critical for identifying resource exhaustion issues and determining when TiFlash is rejecting tasks due to insufficient capacity, which may require scaling or resource reallocation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hard_limit_exceeded_count\"}) by (instance, type, resource_group)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 109, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Estimated Thread Usage and Limit", "group": "Task Scheduler", "used_for": "Used for monitoring TiFlash's task scheduler thread consumption against configured soft and hard limits per resource group. Critical for diagnosing thread exhaustion issues, identifying resource group contention, and preventing task scheduler overload that could impact query execution performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_soft_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_hard_limit\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"global_estimated_thread_usage\"}) by (instance, type, resource_group)", "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"group_entry_count\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1715, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader", "group": "Cluster", "used_for": "Used for monitoring the number of Raft leader regions on each TiFlash node and detecting significant leader transfers. Helps identify load imbalance across TiFlash instances and diagnose unexpected leader migration events (drops of more than 10 leaders within 2 minutes), which may indicate node instability or cluster rebalancing issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m]) < -10" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 38, "dashboard": "DM-Professional-Clinic", "title": "Replication lag", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile replication lag in DM (Data Migration) binlog synchronization tasks. Critical for identifying data synchronization delays between upstream and downstream databases, helping diagnose replication bottlenecks and ensuring data consistency within acceptable time windows.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_lag_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,source_id,task))", "dm_syncer_replication_lag_gauge{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 7, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft read/write proposals", "group": "Raft propose", "used_for": "Used for monitoring the rate of different types of Raft proposals in TiFlash-Proxy, including read operations (local_read, read_index) and write operations (normal, batch). Essential for diagnosing I/O bottlenecks and understanding the distribution of read/write workload patterns in TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|normal|read_index|batch\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2536, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration per server", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile wait time before Raft log entries are applied in TiFlash-Proxy's Raftstore, broken down by server instance. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1975, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft log speed", "group": "Raft propose", "used_for": "Used for monitoring the average rate of Raft log data being proposed in TiFlash-Proxy instances, measured in bytes per second. Helps identify write throughput patterns and potential bottlenecks in Raft log replication to TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 115, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Waiting Duration", "group": "Task Scheduler", "used_for": "Used for monitoring the waiting time of tasks in TiFlash's task scheduler at p80, p90, and p100 percentiles. Essential for diagnosing query queuing issues and resource contention when TiFlash queries experience delays before execution begins.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))", "histogram_quantile(1.00, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1584, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Server is busy", "group": "Errors", "used_for": "Used for monitoring TiFlash-Proxy server overload conditions, including scheduler busy errors, channel congestion, and coprocessor request failures due to capacity limits. Critical for identifying when TiFlash-Proxy instances are overwhelmed by workload, helping diagnose performance degradation and request timeout issues.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type='full'}[2m])) by (instance)", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor error", "group": "Errors", "used_for": "Used for monitoring the rate of coprocessor request errors in TiFlash-Proxy, broken down by instance and error reason. Critical for identifying issues with analytical query processing and diagnosing failures in the coprocessor request handling pipeline between TiDB and TiFlash.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 77, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin apply", "group": "Raft admin", "used_for": "Used for monitoring the rate of successful Raft admin commands in TiFlash-Proxy, broken down by command type (excluding compaction). Essential for tracking administrative operations like region splits, merges, and configuration changes that affect TiFlash's Raft consensus layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status=\"success\", type!=\"compact\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raftstore error", "group": "Errors", "used_for": "Used for monitoring the rate of failed asynchronous requests in TiFlash-Proxy's Raftstore engine, broken down by instance and error status. Critical for identifying storage engine errors and replication issues that could impact TiFlash's ability to sync data from TiKV.", "component": "TIKV", "phenomenon": "error_rate", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"success|all\"}[2m])) by (instance, status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 41, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Propose wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft proposals are processed in TiFlash-Proxy's raftstore, showing p95, p99, and average latencies. Critical for diagnosing Raft replication bottlenecks and identifying when proposals are queuing excessively before being handled, which can indicate raftstore thread saturation or resource contention.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2535, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Apply wait duration", "group": "Raft propose", "used_for": "Used for monitoring the wait time before Raft log entries are applied in TiFlash-Proxy's raftstore, tracking p95, p99, and average latencies. Critical for diagnosing replication lag and identifying bottlenecks in the Raft apply process that could delay data synchronization from TiKV to TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 19, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Internal DDL OPM", "group": "DDL", "used_for": "Used for monitoring the rate of internal DDL operations per minute in TiFlash, broken down by operation type and instance. Essential for tracking schema synchronization activity between TiDB and TiFlash, helping diagnose DDL replication delays or bottlenecks in the columnar storage engine.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)", "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 16, "dashboard": "DM-Professional-Clinic", "title": "SQL process rate", "group": "dump and load ", "used_for": "Used for monitoring the SQL data processing throughput rate during DM's dump and load operations. Helps identify data ingestion bottlenecks and track the speed at which SQL statements are being delivered and processed across different tasks and sources.", "component": "", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m])", "sum(rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[2m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 102, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS", "group": "binlog replication", "used_for": "Used for monitoring the throughput of binlog replication operations in DM (Data Migration), measured as queries per second. Helps identify replication performance bottlenecks and track the rate of different operation types (INSERT/UPDATE/DELETE) being replicated from source to target databases.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval]))by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 76, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Admin proposals", "group": "Raft admin", "used_for": "Used for monitoring the rate of Raft administrative operations (configuration changes and leader transfers) in TiFlash-Proxy. Essential for tracking cluster topology changes and diagnosing issues related to region leadership transfers or membership modifications that may impact TiFlash's data replication stability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 37, "dashboard": "DM-Professional-Clinic", "title": "Remaining time to sync", "group": "binlog replication", "used_for": "Used for monitoring the estimated time remaining for DM (Data Migration) to catch up with the upstream binlog replication lag. Critical for tracking replication progress and identifying when sync delays may impact data freshness or recovery objectives.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_remaining_time{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 1721, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message error", "group": "Errors", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by instance and message type. Critical for identifying communication failures between TiFlash and TiKV that could impact data replication and query execution.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4375, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Perf Context duration", "group": "Raft propose", "used_for": "Used for monitoring the 99th percentile latency of performance context operations during Raft proposal processing in TiFlash-Proxy, including both store and apply phases. Critical for diagnosing slow Raft log replication and identifying performance bottlenecks in TiFlash's consensus layer that may impact write throughput and data synchronization delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply OPM", "group": "DDL", "used_for": "Used for monitoring the rate of schema changes (DDL operations) being applied to TiFlash instances, measured in operations per minute. Helps identify DDL activity patterns and diagnose issues when schema synchronization between TiDB and TiFlash is slow or failing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "DM-Professional-Clinic", "title": "Total bytes for load unit", "group": "dump and load ", "used_for": "Used for monitoring the total data volume processed during DM's load phase, tracking both bytes read from source files and bytes delivered to TiDB. Essential for understanding data migration progress, estimating completion time, and diagnosing throughput bottlenecks in the loading pipeline.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "lightning_row_read_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", "lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 39, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "binlog replication", "used_for": "Used for monitoring abnormal process terminations in DM's binlog replication component. Helps quickly identify and diagnose replication failures by tracking error-triggered exits over 30-minute intervals, essential for maintaining data migration stability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "changes(dm_syncer_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[30m])" ], "dashboard_file": "dm.json" }, { "panel_id": 70, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Check split", "group": "Raft admin", "used_for": "Used for monitoring the rate of region split checks in TiFlash-Proxy's Raft store, categorized by check type. Helps diagnose region split behavior and identify potential issues with region management or split decision-making in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"ignore\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1722, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader drop", "group": "Errors", "used_for": "Used for monitoring the rate of Raft leader region drops in TiFlash-Proxy instances. Essential for detecting leader transfer issues, network instability, or node failures that cause unexpected leadership changes affecting data availability and query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1723, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Leader missing", "group": "Errors", "used_for": "Used for monitoring the number of Raft regions in TiFlash-Proxy that are missing their leaders. Critical for diagnosing replication issues and cluster instability, as missing leaders indicate regions that cannot serve read/write requests until leader election completes.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_leader_missing{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 71, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Check split duration", "group": "Raft admin", "used_for": "Used for monitoring the 99.99th percentile latency of region split checks in TiFlash's Raft proxy layer. Critical for identifying extreme tail latency in split operations that could indicate storage bottlenecks or region management issues affecting TiFlash's data distribution.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 224, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Task Thread Pool Size", "group": "Pipeline Model", "used_for": "Used for monitoring the size of task thread pools in TiFlash's pipeline execution model. Helps diagnose resource allocation issues and thread pool capacity problems when investigating query execution performance or I/O bottlenecks in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_task_thread_pool_size\"}) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema Apply Duration", "group": "DDL", "used_for": "Used for monitoring the latency of DDL schema changes being applied to TiFlash replicas at various percentiles (p80-p999). Critical for diagnosing slow DDL propagation issues and identifying when schema synchronization between TiDB and TiFlash becomes a bottleneck, which can impact query performance on analytical workloads.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster", "type" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", keyspace!=\"\"}[2m])) by (le, type, keyspace))", "sum(tiflash_sync_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"$type\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 110, "dashboard": "DM-Professional-Clinic", "title": "Replication QPS (Sum by task)", "group": "binlog replication", "used_for": "Used for monitoring the binlog replication throughput in DM (Data Migration) tasks, showing both actual finished jobs per second and ideal QPS aggregated by task. Essential for identifying replication performance bottlenecks, comparing actual versus expected throughput, and diagnosing slow data synchronization issues across different migration tasks.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "task" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (task)", "sum(dm_syncer_ideal_qps{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 2, "dashboard": "DM-Professional-Clinic", "title": "DM SQL jobs ", "group": "binlog replication", "used_for": "Used for monitoring the rate of SQL job processing in DM's binlog replication pipeline, showing both completed jobs (finished) and incoming jobs (added) by task, source, schema, and operation type. Essential for identifying replication bottlenecks, job queue buildup, and understanding the throughput of different SQL operation types during data migration.", "component": "", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "interval", "job_id", "source", "target_schema", "task", "type" ], "promql": [ "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", "sum(-rate(dm_syncer_added_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)" ], "dashboard_file": "dm.json" }, { "panel_id": 41, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of write operations in TiFlash storage engine, tracking both storage commands and DeltaTree write block events per second. Essential for identifying write workload patterns, detecting write traffic spikes, and diagnosing TiFlash data ingestion performance issues.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 33, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "CF size", "group": "Server", "used_for": "Used for monitoring the total storage size of different column families (CFs) in TiFlash-Proxy's RocksDB engine. Helps track storage usage patterns and identify which column families are consuming the most disk space, essential for capacity planning and detecting abnormal data growth.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Time used by level", "group": "Unified Read Pool", "used_for": "Used for monitoring the time distribution across different priority levels in TiFlash's unified read pool. Helps identify if certain priority levels are consuming disproportionate resources, which can indicate workload imbalances or priority queue bottlenecks affecting read performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}[2m])) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 64, "dashboard": "DM-Professional-Clinic", "title": "DML conflict detect duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of DML conflict detection during binlog replication in DM (Data Migration) tasks at p90, p95, and p99 percentiles. Critical for diagnosing replication slowdowns caused by conflict detection overhead, especially when multiple concurrent writes target the same data and require conflict resolution.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 38, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Amplification", "group": "Storage", "used_for": "Used for monitoring the write amplification ratio in TiFlash storage, which measures how much data is actually written to disk compared to the logical data received. Critical for diagnosing storage efficiency issues and identifying excessive disk I/O caused by compaction, replication, or inefficient write patterns that can impact TiFlash performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[10m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[30m])\n)", "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)", "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4196, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Level 0 chance", "group": "Unified Read Pool", "used_for": "Used for monitoring the probability of tasks being scheduled at level 0 (highest priority) in TiFlash's unified read pool. Helps diagnose read request prioritization issues and identify when the system is consistently processing high-priority requests, which may indicate resource contention or workload imbalance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_multilevel_level0_chance{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=\"unified-read-pool\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1705, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the total storage size of each TiFlash-Proxy instance's RocksDB engine. Essential for tracking disk space usage, capacity planning, and identifying instances approaching storage limits that may require scaling or data cleanup.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 22, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Channel full", "group": "Server", "used_for": "Used for monitoring the rate of channel full events in TiFlash-Proxy, indicating when internal message queues reach capacity. Critical for diagnosing backpressure issues, message processing bottlenecks, and potential data synchronization delays between TiFlash and TiKV.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_channel_full_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4198, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Running tasks", "group": "Unified Read Pool", "used_for": "Used for monitoring the number of actively running tasks in TiFlash-Proxy's unified read pool at any given time. Helps identify read pool saturation and potential bottlenecks when tasks accumulate, indicating insufficient read processing capacity or slow query execution.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 40, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the rate of read tasks processed per second by each TiFlash instance. Helps identify read workload distribution across TiFlash nodes and diagnose performance bottlenecks when read throughput is unbalanced or unexpectedly low.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 253, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (bytes)", "group": "Storage", "used_for": "Used for monitoring the write throughput of TiFlash storage subtasks, broken down by operation type. Helps identify write performance bottlenecks and understand which storage operations are consuming the most bandwidth in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 255, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SubTasks Write Throughput (rows)", "group": "Storage", "used_for": "Used for monitoring the row-level write throughput of TiFlash storage subtasks, broken down by operation type. Essential for analyzing TiFlash write performance, identifying bottlenecks in data ingestion from TiKV, and diagnosing issues when columnar replicas fall behind or experience slow synchronization.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_throughput_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4, "dashboard": "DM-Professional-Clinic", "title": "binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being processed per second during DM (Data Migration) replication. Essential for tracking replication throughput and identifying bottlenecks or slowdowns in the binlog transformation pipeline across different tasks and stages.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "interval", "job_id", "source", "stage", "task" ], "promql": [ "sum by (instance,task,stage) (rate(dm_syncer_binlog_transform_cost_count{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\", source_id=~\"$source\",stage=~\"$stage\"}[$interval]))" ], "dashboard_file": "dm.json" }, { "panel_id": 75, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Active written leaders", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of TiFlash-Proxy raft regions actively receiving write operations per instance. Helps identify write hotspots and uneven write distribution across TiFlash nodes, which can indicate data skew or replication issues.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1481, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size", "group": "Server", "used_for": "Used for monitoring the approximate size distribution of Raft regions stored in TiFlash-Proxy, showing p95, p99, and average values. Essential for identifying oversized regions that could impact I/O performance, memory usage, and replication efficiency in TiFlash's columnar storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 8, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async request error", "group": "Storage", "used_for": "Used for monitoring the error rate of asynchronous storage requests in TiFlash-Proxy's storage engine. Critical for identifying storage layer failures and diagnosing data access issues that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", status!~\"all|success\"}[2m])) by (status)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 60, "dashboard": "DM-Professional-Clinic", "title": "transaction/ddl query execution latency", "group": "binlog replication", "used_for": "Used for monitoring the P90, P95, and P99 latency of transaction and DDL query execution during binlog replication in DM (Data Migration). Critical for identifying replication lag issues and diagnosing slow transaction processing that could impact data synchronization performance between upstream and downstream databases.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.90, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 58, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written bytes", "group": "Server", "used_for": "Used for monitoring the average bytes written per region in TiFlash-Proxy instances. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 130, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of TiFlash's internal storage maintenance operations, including delta merge, segment merge, and segment split tasks. Essential for diagnosing TiFlash storage performance issues and understanding background compaction activity that may impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage command total", "group": "Storage", "used_for": "Used for monitoring the rate of different storage command types executed in TiFlash-Proxy. Helps identify command distribution patterns and diagnose issues when specific storage operations (like get, scan, prewrite) show abnormal rates or spikes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 42, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum latency of internal storage subtasks in TiFlash, such as delta merge operations and background compaction tasks. Critical for diagnosing TiFlash storage engine performance issues and identifying bottlenecks in background maintenance operations that could impact query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 3638, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Approximate Region size Histogram", "group": "Server", "used_for": "Used for monitoring the distribution of region sizes in TiFlash-Proxy's Raft store. Helps identify region size imbalances and diagnose issues related to region splitting, merging, or uneven data distribution that could impact I/O performance and storage efficiency.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 39, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Small Internal Tasks OPS", "group": "Storage", "used_for": "Used for monitoring the throughput of small internal storage tasks in TiFlash, excluding major operations like delta merge, segment merge, and segment split. Helps identify the activity level of background maintenance tasks and diagnose potential bottlenecks in TiFlash's internal storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 57, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region average written keys", "group": "Server", "used_for": "Used for monitoring the average number of keys written per region in TiFlash-Proxy. Helps identify uneven write distribution across regions and diagnose potential hotspot issues or data skew in TiFlash storage.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async write duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous write operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing TiFlash write performance bottlenecks and identifying when data ingestion from TiKV is experiencing delays or degradation.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 35, "dashboard": "DM-Professional-Clinic", "title": "skipped binlog event QPS", "group": "binlog replication", "used_for": "Used for monitoring the rate of binlog events being skipped during DM (Data Migration) replication, broken down by rows and query types. Essential for identifying data filtering effectiveness and diagnosing potential data synchronization gaps when binlog filter rules are applied.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"rows\"}[$interval])", "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"query\"}[$interval])" ], "dashboard_file": "dm.json" }, { "panel_id": 193, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler pending commands", "group": "Scheduler", "used_for": "Used for monitoring the total number of pending scheduler commands in TiFlash-Proxy instances. Helps identify scheduling bottlenecks and command queue buildup that could indicate resource contention or processing delays in TiFlash's Raft proxy layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_contex_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 167, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler", "used_for": "Used for monitoring the total number of scheduler operations across different stages in TiFlash-Proxy, including both normal operations and \"too busy\" rejections. Essential for diagnosing scheduler bottlenecks and understanding when TiFlash-Proxy is overloaded or rejecting requests due to resource constraints.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3637, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written keys", "group": "Server", "used_for": "Used for monitoring the rate of keys written to regions in TiFlash-Proxy's storage layer. Helps diagnose write throughput issues and identify regions experiencing heavy write activity that may impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 43, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM syncer during data migration. Critical for identifying replication lag and potential data synchronization delays, helping diagnose whether the syncer is falling behind in processing binlog events.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"master\"} - ON(task,source_id) dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 22, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "File Open OPS", "group": "Storage", "used_for": "Used for monitoring the rate of file open operations and failures in TiFlash storage layer. Helps diagnose file descriptor exhaustion, storage I/O issues, or excessive file access patterns that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 15, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage async snapshot duration", "group": "Storage", "used_for": "Used for monitoring the latency of asynchronous snapshot operations in TiFlash's storage engine at p95 and p99 percentiles. Critical for diagnosing slow data replication from TiKV to TiFlash and identifying performance bottlenecks in snapshot processing that could impact analytical query freshness.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler command duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler commands in TiFlash-Proxy at p95 and p99 percentiles. Essential for diagnosing performance bottlenecks in TiFlash's transaction scheduling layer, particularly when investigating slow data replication or command processing issues between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 50, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Opened File Count", "group": "Storage", "used_for": "Used for monitoring the number of open file descriptors in TiFlash instances, including total open files, files opened for writing, and files opened for reading. Critical for detecting file descriptor leaks and preventing \"too many open files\" errors that can cause TiFlash crashes or performance degradation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler priority commands", "group": "Scheduler", "used_for": "Used for monitoring the rate of scheduler commands by priority level in TiFlash-Proxy. Helps identify priority distribution of I/O operations and diagnose whether high-priority commands are being properly processed or if low-priority commands are overwhelming the scheduler.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (priority)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3636, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Region written bytes", "group": "Server", "used_for": "Used for monitoring the rate of bytes written to regions in TiFlash-Proxy's storage layer. Helps diagnose write throughput issues, identify heavy write workloads, and detect abnormal I/O patterns that may impact TiFlash replication performance.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_proxy_tikv_region_written_bytes_bucket[1m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 131, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Large Internal Tasks Duration", "group": "Storage", "used_for": "Used for monitoring the maximum duration of large internal storage tasks in TiFlash, such as delta merge, delta merge background, and place index operations. Critical for diagnosing TiFlash storage layer performance issues and identifying when background maintenance tasks are taking unusually long to complete.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1, "dashboard": "DM-Professional-Clinic", "title": "statement execution latency - 90", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile execution latency of binlog replication statements in DM (Data Migration), broken down by transaction phases (begin, statement execution, and commit). Critical for identifying performance bottlenecks in the replication pipeline and diagnosing slow transaction processing that could cause replication lag.", "component": "", "phenomenon": "latency", "object": "query", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"begin\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"stmt\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"commit\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"rollback\", instance=~\"$instance\"}[2m])) by (le,task,instance))" ], "dashboard_file": "dm.json" }, { "panel_id": 62, "dashboard": "DM-Professional-Clinic", "title": "DML queue remain length", "group": "binlog replication", "used_for": "Used for monitoring the number of DML (Data Manipulation Language) events queued and waiting to be replicated in DM's binlog synchronization pipeline. Critical for identifying replication bottlenecks and backlog issues when the queue length grows, indicating that DML processing cannot keep up with the incoming binlog events.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "task" ], "promql": [ "sum(dm_syncer_queue_size{job_id=~\"$job_id\",task=~\"$task\"}) by (task)" ], "dashboard_file": "dm.json" }, { "panel_id": 3718, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch ratio", "group": "Server", "used_for": "Used for monitoring the batch efficiency of requests processed by TiFlash-Proxy, showing both average and P99 batch ratios by request type. Essential for identifying request batching inefficiencies that could impact TiFlash I/O performance and diagnosing whether requests are being properly grouped for optimal processing.", "component": "TIFLASH", "phenomenon": "io", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3834, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler writing bytes", "group": "Scheduler", "used_for": "Used for monitoring the volume of data being written by the scheduler in TiFlash-Proxy instances. Helps identify write bottlenecks, uneven write distribution across instances, and potential memory pressure issues in the scheduler's write pipeline.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_scheduler_writing_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3720, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Request batch input", "group": "Server", "used_for": "Used for monitoring the batch size of incoming requests to TiFlash-Proxy, showing both average and p99 values by request type. Helps identify request batching efficiency and detect anomalies where requests aren't being properly batched, which could impact TiFlash query performance and resource utilization.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 59, "dashboard": "DM-Professional-Clinic", "title": "dispatch binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of dispatching binlog events in DM (Data Migration) replication tasks at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when event dispatch delays are impacting replication lag between source and target databases.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 106, "dashboard": "DM-Professional-Clinic", "title": "binlog event row number", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile of row counts per binlog event during DM (Data Migration) replication. Helps identify binlog events with unusually large row batches that may cause replication lag or memory pressure in the sync unit.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_row_bucket{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source))" ], "dashboard_file": "dm.json" }, { "panel_id": 58, "dashboard": "DM-Professional-Clinic", "title": "binlog event cost duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of binlog event transformation at different stages (p90, p95, p99 percentiles) in DM (Data Migration) syncer tasks. Essential for diagnosing binlog replication performance bottlenecks and identifying which transformation stage is causing delays in data synchronization from source to target.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task,stage))" ], "dashboard_file": "dm.json" }, { "panel_id": 168, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler stage total", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of scheduler operations across different stages in TiFlash-Proxy's transaction scheduler, including tracking when the scheduler becomes too busy to accept requests. Essential for diagnosing transaction processing bottlenecks and identifying which scheduler stage (snapshot, async_write, etc.) is causing performance degradation or resource saturation.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m]))", "sum(rate(tiflash_proxy_tikv_scheduler_stage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (stage)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 46, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Write OPS", "group": "Storage", "used_for": "Used for monitoring disk write operations per second in TiFlash storage layer, including PSM (PageStorage Manager) write calls, page writes, and file descriptor writes. Essential for diagnosing TiFlash write throughput bottlenecks and identifying abnormal disk I/O patterns that may impact columnar storage performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 373, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys written", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys written per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Essential for identifying write-heavy operations and diagnosing performance issues related to excessive key modifications in specific scheduler commands.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 36, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of reading binlog events from upstream MySQL sources in DM (Data Migration) replication tasks. Critical for diagnosing binlog replication delays and identifying slow binlog read operations that could bottleneck data synchronization performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 107, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message failed", "group": "gRPC", "used_for": "Used for monitoring the rate of failed gRPC messages in TiFlash-Proxy, broken down by message type. Essential for identifying communication issues between TiFlash and other cluster components, helping diagnose network problems or service degradation.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2534, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "raft message batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile batch size of Raft messages in TiFlash-Proxy's gRPC communication. Helps diagnose message batching efficiency and identify potential network or replication bottlenecks when batch sizes are abnormally small or large.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 45, "dashboard": "DM-Professional-Clinic", "title": "unsynced tables", "group": "binlog replication", "used_for": "Used for monitoring the number of tables that have not yet been synchronized in DM (Data Migration) binlog replication tasks. Critical for identifying replication lag at the table level and diagnosing incomplete data migration scenarios.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_unsynced_table_number{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 194, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler latch wait duration", "group": "Scheduler - $command", "used_for": "Used for monitoring the latency of scheduler latch waiting in TiFlash-Proxy at p95 and p99 percentiles. Critical for diagnosing transaction conflicts and contention issues when multiple operations compete for the same keys, helping identify performance bottlenecks caused by lock waiting in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 47, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Disk Read OPS", "group": "Storage", "used_for": "Used for monitoring the rate of disk read operations in TiFlash storage layer, including PSM (PageStorage Manager) read I/O calls, page reads, and file descriptor reads. Essential for diagnosing TiFlash read performance bottlenecks and identifying excessive disk I/O that may impact query performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 60, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write flow", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash's write throughput across different write paths, including file descriptor writes, PageStorage (PSM) writes, and background writes. Essential for diagnosing TiFlash storage I/O bottlenecks, understanding write workload distribution, and identifying abnormal write patterns that may impact data ingestion performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 98, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile latency of gRPC message handling in TiFlash-Proxy. Critical for identifying communication bottlenecks between TiFlash and TiKV, helping diagnose slow data replication or query execution issues caused by inter-component network delays.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 95, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC message count", "group": "gRPC", "used_for": "Used for monitoring the rate of gRPC messages processed by TiFlash-Proxy, broken down by message type. Helps identify communication patterns between TiFlash and TiKV, and diagnose issues with inter-component RPC traffic or message processing bottlenecks.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!=\"kv_gc\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 107, "dashboard": "DM-Professional-Clinic", "title": "Flushed Checkpoint Time Interval", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile time interval between checkpoint flushes during binlog replication in DM (Data Migration). Critical for diagnosing checkpoint persistence delays and identifying potential data loss risks or replication lag issues caused by infrequent checkpoint updates.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90 ], "variables": [ "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_flush_checkpoints_time_interval_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task,worker))" ], "dashboard_file": "dm.json" }, { "panel_id": 59, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read flow", "group": "Storage", "used_for": "Used for monitoring the read I/O throughput in TiFlash storage layer, including file descriptor reads, PageStorage (PSM) reads, and background reads. Essential for diagnosing read performance bottlenecks, identifying excessive disk I/O, and understanding storage access patterns during query execution.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 44, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between relay and syncer", "group": "binlog replication", "used_for": "Used for monitoring the binlog file position gap between the relay log and syncer components in DM (Data Migration). Helps identify replication lag by showing how many binlog files the syncer is behind the relay, which is critical for diagnosing data synchronization delays and potential bottlenecks in the migration pipeline.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id", "task" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"} - ON(instance) group_right dm_syncer_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", task=~\"$task\", node=\"syncer\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 61, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the size distribution of binlog events being replicated by DM (Data Migration) workers at p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that may cause replication delays, memory pressure, or network bottlenecks during data synchronization from MySQL to TiDB.", "component": "", "phenomenon": "io", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task", "worker" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,worker,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[2m])) by (le,worker,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 237, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexError", "group": "Storage", "used_for": "Used for monitoring the rate of delta index errors in TiFlash's DeltaTree storage engine. Critical for identifying data integrity issues or corruption in TiFlash's columnar storage layer that could affect query accuracy.", "component": "TIFLASH", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 52, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "FSync Status", "group": "Storage", "used_for": "Used for monitoring the frequency and latency of fsync operations in TiFlash storage. Critical for diagnosing disk I/O performance issues and identifying when data persistence operations are causing write bottlenecks or slowdowns.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2532, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Average gRPC messge duration", "group": "gRPC", "used_for": "Used for monitoring the average latency of gRPC messages between TiFlash and TiKV, broken down by message type. Essential for diagnosing communication bottlenecks and identifying which specific gRPC operations are experiencing delays in the TiFlash-TiKV interaction layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 67, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Current Data Management Tasks", "group": "Storage", "used_for": "Used for monitoring the number of active data management operations in TiFlash's DeltaTree storage engine, including delta merges, segment splits, and segment merges. Essential for diagnosing TiFlash storage performance issues and understanding background compaction workload that affects query and write performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 63, "dashboard": "DM-Professional-Clinic", "title": "add job duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of adding binlog replication jobs to DM syncer queues at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when job queuing delays impact replication performance.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "queueNo", "source", "task", "type" ], "promql": [ "histogram_quantile(0.90, sum (rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.95, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "histogram_quantile(0.99, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))" ], "dashboard_file": "dm.json" }, { "panel_id": 46, "dashboard": "DM-Professional-Clinic", "title": "shard lock resolving", "group": "binlog replication", "used_for": "Used for monitoring the number of shard DDL locks currently being resolved in DM (Data Migration) during binlog replication. Critical for diagnosing coordination issues when merging sharded tables, as prolonged lock resolving indicates DDL synchronization delays or conflicts across multiple upstream MySQL instances.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "job_id", "source", "task" ], "promql": [ "dm_syncer_shard_lock_resolving{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 560, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations in TiFlash-Proxy's scheduler, broken down by operation tags. Helps diagnose scan performance issues and understand the characteristics of different scan types (e.g., processed keys, skipped keys) during command execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 65, "dashboard": "DM-Professional-Clinic", "title": "skip event duration", "group": "binlog replication", "used_for": "**Used For:**\n\nUsed for monitoring the latency of skipping binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing performance issues when DM filters or skips certain binlog events based on configured rules, helping identify if event filtering is causing replication delays.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "interval", "job_id", "source", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.95, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "histogram_quantile(0.99, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))" ], "dashboard_file": "dm.json" }, { "panel_id": 2533, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC batch size", "group": "gRPC", "used_for": "Used for monitoring the 99th percentile and average batch sizes of gRPC requests and responses in TiFlash-Proxy. Essential for diagnosing gRPC communication efficiency and identifying potential batching issues that could impact data replication performance between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))", "sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 675, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [lock]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate of key-value scan operations on the lock column family within TiFlash-Proxy's scheduler for specific commands. Helps diagnose lock-related scanning performance and identify potential lock contention or scanning bottlenecks in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"lock\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 829, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [write]", "group": "Scheduler - $command", "used_for": "Used for monitoring the rate and distribution of key-value scan operations on TiFlash-Proxy's write column family during scheduler command execution. Essential for diagnosing write-path performance issues and understanding scan patterns that may impact TiFlash write throughput.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"write\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 61, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Raft store CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash's Raft store threads, which handle Raft consensus protocol operations for data replication. Essential for diagnosing performance bottlenecks in TiFlash's data synchronization layer and identifying instances with excessive Raft processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 105, "dashboard": "DM-Professional-Clinic", "title": "replication transaction batch", "group": "binlog replication", "used_for": "Used for monitoring the 90th percentile batch size of replicated transactions in DM's binlog replication process. Helps identify transaction batching efficiency and diagnose replication throughput issues when batch sizes are unexpectedly small or large.", "component": "", "phenomenon": "io", "object": "transaction", "percentile": [ 90 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_replication_transaction_batch_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task,queueNo,type))" ], "dashboard_file": "dm.json" }, { "panel_id": 266, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Pending Rate and Duration", "group": "Storage", "used_for": "Used for monitoring the rate of I/O operations waiting in TiFlash's I/O limiter queue, broken down by operation type. Essential for identifying I/O throttling issues and diagnosing storage performance bottlenecks when TiFlash experiences read/write delays due to rate limiting.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter_pending_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 79, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Async apply CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's async apply threads, which handle Raft log application to the storage engine. Essential for diagnosing write performance bottlenecks and identifying whether apply threads are saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"apply_[0-9]+\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 20, "dashboard": "DM-Professional-Clinic", "title": "storage capacity", "group": "relay log", "used_for": "Used for monitoring the total storage capacity allocated for DM (Data Migration) relay logs. Helps administrators track available disk space and plan capacity before relay log storage becomes full, which could halt data replication.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"capacity\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 830, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler scan details [default]", "group": "Scheduler - $command", "used_for": "Used for monitoring the distribution of key-value scan operations in TiFlash-Proxy's scheduler for the default column family. Helps diagnose scan performance issues by breaking down scan operations by tag, useful when investigating slow queries or identifying inefficient scan patterns in TiFlash storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", req=\"$command\", cf=\"default\"}[2m])) by (tag)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 36, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99% Handle snapshot duration", "group": "Snapshot", "used_for": "Used for monitoring the 99th percentile latency of snapshot operations in TiFlash-Proxy, including sending, applying, and generating snapshots. Critical for diagnosing slow replica synchronization and identifying performance bottlenecks in TiFlash's Raft snapshot handling process.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"apply\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"generate\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 109, "dashboard": "DM-Professional-Clinic", "title": " transform binlog event duration", "group": "binlog replication", "used_for": "Used for monitoring the latency of transforming binlog events during DM (Data Migration) replication at p90, p95, and p99 percentiles. Critical for diagnosing binlog processing bottlenecks and identifying when event transformation becomes a performance constraint in the replication pipeline.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "interval", "job_id", "task" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))" ], "dashboard_file": "dm.json" }, { "panel_id": 38, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Snapshot state count", "group": "Snapshot", "used_for": "Used for monitoring the total count of Raft snapshots by type (sending/receiving/applying) in TiFlash-Proxy. Helps diagnose snapshot transfer issues, replica synchronization problems, and identify excessive snapshot activity that may impact cluster performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 64, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's scheduler worker threads, which handle task scheduling and coordination. Essential for identifying scheduler bottlenecks, thread contention issues, or abnormal CPU consumption that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 86, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Current Pending Count", "group": "Storage", "used_for": "Used for monitoring the number of write requests currently queued in TiFlash's I/O limiter, broken down by rate limiter, background writes, and foreground writes. Essential for diagnosing write throttling issues and identifying when storage I/O limits are causing request backlog in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 78, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Coprocessor CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash coprocessor threads across different priority levels (normal, high, and low). Essential for diagnosing TiFlash query processing performance issues and identifying CPU bottlenecks in analytical workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_normal.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 35, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Rate snapshot message", "group": "Snapshot", "used_for": "Used for monitoring the rate of Raft snapshot messages sent by TiFlash-Proxy nodes. Essential for tracking data replication activity and diagnosing issues with region snapshot transfers, which typically occur during node recovery, scaling operations, or when replicas fall significantly behind.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 269, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Scheduling", "group": "Storage", "used_for": "Used for monitoring TiFlash read thread scheduling behavior and resource utilization patterns. Helps diagnose read performance bottlenecks by tracking thread exhaustion, active segment limits, cache hits, and scheduling efficiency in the storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 132, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Data Sharing", "group": "Storage", "used_for": "**Used For:**\n\nUsed for monitoring TiFlash's data sharing cache mechanism, which enables multiple read threads to share data blocks and reduce redundant I/O operations. Helps diagnose cache efficiency by tracking cache hit rates, miss rates, and cache addition operations, essential for optimizing TiFlash read performance and memory utilization.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"add_cache_stale|add_cache_succ|add_cache_total_bytes_limit|add_cache_reach_count_limit\"}[2m])) by (type)", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))", "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))/sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 105, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "gRPC poll CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of gRPC polling threads in TiFlash-Proxy instances. Essential for identifying excessive CPU consumption in gRPC communication layers, which may indicate network bottlenecks or inefficient request handling between TiFlash and other TiDB components.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"grpc.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4287, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Unified read pool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's unified read pool threads across instances. Essential for identifying read workload bottlenecks and determining if the unified read pool requires resource adjustment when query performance degrades.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"unf_rd_pool.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 29, "dashboard": "DM-Professional-Clinic", "title": "fail to write relay log", "group": "relay log", "used_for": "Used for monitoring relay log write failures in DM (Data Migration) workers. Critical for diagnosing data replication interruptions caused by I/O errors when persisting binlog events to local relay logs.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_write_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 169, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Effectiveness of Mark Cache", "group": "Storage", "used_for": "Used for monitoring the hit and miss rates of TiFlash's mark cache, which stores metadata about data block boundaries in columnar storage. Essential for evaluating cache efficiency and diagnosing performance issues related to metadata lookups during query execution.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_system_profile_event_MarkCacheMisses{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "max(tiflash_system_profile_event_MarkCacheHits{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1908, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Storage ReadPool CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU utilization of TiFlash-Proxy's storage read thread pools across different priority levels (normal, high, and low). Essential for diagnosing read performance bottlenecks and identifying whether specific priority queues are saturated or experiencing CPU contention.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_norm.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_high.*\"}[2m])) by (instance)", "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"store_read_low.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2531, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "GC worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's garbage collection worker threads. Essential for diagnosing GC performance issues and identifying instances where excessive GC activity may be impacting overall TiFlash performance.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"gc_worker.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 59, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different worker threads in TiFlash-Proxy. Helps identify worker thread activity patterns and diagnose task processing bottlenecks or imbalances across worker pools.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 31, "dashboard": "DM-Professional-Clinic", "title": "relay log data corruption", "group": "relay log", "used_for": "Used for detecting data corruption events in DM (Data Migration) relay logs. Critical for identifying data integrity issues during binlog replication that could lead to inconsistent data migration or replication failures.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_data_corruption{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 292, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Thread Internal Duration", "group": "Storage", "used_for": "Used for monitoring the internal processing latency of TiFlash read threads at various percentiles (p80-p999). Critical for diagnosing TiFlash query performance bottlenecks and identifying whether slow analytical queries are caused by internal read thread processing delays rather than external factors.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.999, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 42, "dashboard": "DM-Professional-Clinic", "title": "binlog file gap between master and relay", "group": "relay log", "used_for": "Used for monitoring the binlog file position gap between the upstream MySQL master and DM's relay log. Helps identify replication lag at the file level and diagnose relay log synchronization delays or issues in data migration tasks.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"} - ON(instance) dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 30, "dashboard": "DM-Professional-Clinic", "title": "process exits with error", "group": "relay log", "used_for": "Used for monitoring the count of relay log process abnormal exits in DM (Data Migration). Critical for identifying relay log stability issues and process crashes that could interrupt data replication from MySQL binlog to downstream TiDB.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_exit_with_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 21, "dashboard": "DM-Professional-Clinic", "title": "storage remain", "group": "relay log", "used_for": "**Used For:**\n\nUsed for monitoring the available disk space remaining for DM relay logs. Critical for preventing relay log write failures due to insufficient storage and ensuring continuous data replication from upstream MySQL to downstream TiDB.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"available\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 44, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot size", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of snapshot sizes in TiFlash-Proxy's Raft replication process. Critical for identifying abnormally large snapshots that could cause memory pressure, network congestion, or slow replica synchronization during TiFlash node recovery or scaling operations.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 88, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Snapshots", "group": "Storage", "used_for": "Used for monitoring the number of active read snapshots and segment read tasks in TiFlash's storage layer. Essential for diagnosing MVCC snapshot accumulation issues, memory pressure from long-running queries, and identifying potential read performance bottlenecks caused by excessive concurrent read operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 43, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "99.99% Snapshot KV count", "group": "Snapshot", "used_for": "Used for monitoring the 99.99th percentile of key-value pair counts in TiFlash-Proxy snapshots. Helps identify extreme cases where snapshots contain unusually large numbers of KV pairs, which can indicate data skew issues or potential performance bottlenecks during snapshot processing.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [ 9999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1395, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Worker pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy worker queues, broken down by worker name. Essential for identifying task queue bottlenecks and worker thread saturation that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_worker_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 84, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "I/O Limiter Throughput", "group": "Storage", "used_for": "Used for monitoring the I/O throughput rate of TiFlash's I/O limiter, broken down by operation type (read/write). Essential for identifying I/O bottlenecks, verifying rate limiting effectiveness, and diagnosing storage performance issues when TiFlash queries experience slowdowns.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 1877, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool pending tasks", "group": "Task", "used_for": "Used for monitoring the rate of pending tasks in TiFlash-Proxy's future pool, broken down by pool name. Helps identify task queue buildup and potential thread pool saturation issues that could impact TiFlash query processing performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 168, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Schema of Column File", "group": "Storage", "used_for": "Used for monitoring TiFlash's column file schema cache performance, tracking cache size, hit rate, and eviction patterns. Essential for diagnosing schema cache efficiency issues that may impact TiFlash query performance and memory usage.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)", "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[2m])) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)", "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 69, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "RocksDB CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of RocksDB threads in TiFlash-Proxy instances. Essential for identifying CPU bottlenecks in the storage engine layer and diagnosing performance issues related to data compaction, writes, and reads in TiFlash's underlying RocksDB.", "component": "TIKV", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 1876, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "FuturePool handled tasks", "group": "Task", "used_for": "Used for monitoring the rate of tasks handled by different thread pools in TiFlash-Proxy's future pool executor. Helps identify thread pool utilization patterns and diagnose task processing bottlenecks or imbalances across different worker pools.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 67, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Background worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy background worker threads across instances. Essential for identifying background task overhead and diagnosing performance issues caused by maintenance operations like compaction, cleanup, or internal housekeeping tasks.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"background.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 238, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Mark Cache and Minmax Index Cache Memory Usage", "group": "Storage", "used_for": "Used for monitoring memory consumption of TiFlash's data indexing structures, including mark cache (data location markers), minmax index files, and delta index cache. Essential for diagnosing TiFlash memory pressure issues and optimizing cache configurations when analytical query performance degrades.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "tiflash_system_asynchronous_metric_RNDeltaIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 6, "dashboard": "DM-Professional-Clinic", "title": "binlog pos", "group": "relay log", "used_for": "Used for monitoring the binlog position tracking in DM's relay log processing, comparing the master's binlog position against the relay log's current position. Essential for identifying replication lag and detecting if relay log processing is falling behind the upstream MySQL master.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 7, "dashboard": "DM-Professional-Clinic", "title": "binlog file index", "group": "relay log", "used_for": "Used for tracking the current binlog file index being processed by DM's relay log component. Helps identify relay log progression, detect synchronization lag between master and relay nodes, and diagnose issues where relay log processing may be stuck or falling behind the upstream MySQL binlog position.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 68, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read Index Worker CPU", "group": "Thread CPU", "used_for": "Used for monitoring CPU usage of TiFlash-Proxy's Read Index worker threads, which handle Raft read index requests to ensure read consistency. Essential for diagnosing performance bottlenecks when TiFlash queries experience high latency due to read index processing overhead.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", name=~\"ReadIndexWkr_.*\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 70, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Throughput", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage layer throughput, separating write/ingest operations from delta management operations (compaction, merge). Critical for diagnosing write stall issues by identifying whether bottlenecks stem from incoming writes or background delta management processes.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[2m]))", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 28, "dashboard": "DM-Professional-Clinic", "title": "fail to read binlog from master", "group": "relay log", "used_for": "Used for monitoring the count of errors encountered when DM (Data Migration) relay log fails to read binlog from the upstream MySQL master. Critical for diagnosing replication interruptions and connectivity issues between DM relay and the source database.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "job_id" ], "promql": [ "dm_relay_read_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 62, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Stall Duration", "group": "Storage Write Stall", "used_for": "Used for monitoring the 99th percentile duration of write stalls in TiFlash's storage engine. Critical for diagnosing write performance degradation caused by background compaction, memory pressure, or disk I/O bottlenecks that temporarily block incoming writes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, instance))", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type, instance) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 89, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Throughput By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring write and ingest throughput (bytes per second) for each TiFlash instance. Essential for identifying write performance bottlenecks, uneven write distribution across instances, and diagnosing storage write stall issues in TiFlash's columnar storage engine.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write\"}[2m])) by (instance)", "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 137, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write & Delta Management Total", "group": "Storage Write Stall", "used_for": "Used for monitoring TiFlash storage I/O throughput, separating write/ingest operations from delta management operations (compaction, GC). Critical for diagnosing write stall issues by identifying whether bottlenecks stem from incoming writes or background delta management tasks.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})", "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "DM-Professional-Clinic", "title": "write relay log duration", "group": "relay log", "used_for": "Used for monitoring the write latency of relay logs in DM (Data Migration) at different percentiles (p90, p95, p99). Critical for diagnosing relay log write performance issues and identifying bottlenecks in the data replication pipeline that could impact migration throughput.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 2108, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads state", "group": "Threads", "used_for": "Used for monitoring the state distribution of threads in TiFlash-Proxy instances, showing how many threads are in different states (running, idle, blocked, etc.) per instance. Essential for diagnosing thread pool saturation, identifying I/O bottlenecks, and detecting thread starvation issues that could impact TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, state)", "sum(tiflash_proxy_threads_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "DM-Professional-Clinic", "title": "read binlog event duration", "group": "relay log", "used_for": "Used for monitoring the latency of reading binlog events from the relay log in DM (Data Migration), showing p90, p95, and p99 percentiles. Critical for diagnosing slow binlog processing issues and identifying performance bottlenecks in the relay log reading stage that could impact data replication lag.", "component": "", "phenomenon": "latency", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 2258, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Threads IO", "group": "Threads", "used_for": "Used for monitoring I/O throughput (bytes per second) across different TiFlash-Proxy threads, filtered to show only threads with I/O rates exceeding 1KB/s. Essential for identifying which threads are performing heavy disk operations and diagnosing I/O bottlenecks or imbalanced workload distribution in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (name, io) > 1024" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2660, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Voluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring voluntary context switches per second in TiFlash-Proxy threads, filtered to show only rates exceeding 200 switches/second. High voluntary context switch rates typically indicate threads frequently waiting for I/O operations or locks, helping diagnose I/O bottlenecks and thread contention issues in TiFlash-Proxy.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_voluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 200" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2661, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Thread Nonvoluntary Context Switches", "group": "Threads", "used_for": "Used for monitoring involuntary context switches in TiFlash-Proxy threads, which occur when the OS forcibly preempts threads due to resource contention or time slice expiration. High rates (>50/sec) indicate CPU saturation, thread contention, or I/O blocking issues that can degrade TiFlash query performance.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_thread_nonvoluntary_context_switches{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, name) > 50" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 86, "dashboard": "DM-Professional-Clinic", "title": "number of dm-masters start leader components per minute", "group": "HA", "used_for": "Used for monitoring the frequency of DM-master leader component initialization events per minute. Helps diagnose high availability issues, unexpected leader switches, or cluster instability when the counter shows frequent leader component restarts.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_master_start_leader_counter" ], "dashboard_file": "dm.json" }, { "panel_id": 90, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Write Command OPS By Instance", "group": "Storage Write Stall", "used_for": "Used for monitoring the rate of write operations and storage commands per TiFlash instance. Helps identify write throughput imbalances across instances and diagnose storage write stall issues by tracking both DM write blocks and storage command execution rates.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)", "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 128, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Disk Usage", "group": "PageStorage", "used_for": "Used for monitoring disk space usage and efficiency of TiFlash's PageStorage blob storage. Helps identify disk space issues by tracking total disk bytes, valid data bytes, and space utilization ratio, essential for capacity planning and detecting storage fragmentation or waste.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})) by (instance)", "tiflash_system_asynchronous_metric_LogDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Scheduler keys read", "group": "Scheduler - $command", "used_for": "Used for monitoring the number of keys read per scheduler command in TiFlash-Proxy at p95, p99, and average percentiles. Helps diagnose I/O efficiency issues and identify commands that scan excessive keys, which may indicate inefficient queries or potential performance bottlenecks in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [ 95, 99 ], "variables": [ "command", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"$command\"}[2m])) " ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 138, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get operations", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB read operation efficiency in TiFlash-Proxy, tracking hit rates across different storage layers (memtable, block cache, and L0 SST files). Essential for diagnosing read performance issues and understanding how effectively data is being served from memory versus disk in the storage engine.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_get_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 83, "dashboard": "DM-Professional-Clinic", "title": "number of workers in different state", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the distribution of DM (Data Migration) workers across different operational states in the cluster. Essential for diagnosing high availability issues, identifying stuck or failed workers, and verifying proper worker state transitions during failover scenarios.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "count_values(\"state\", dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 25, "dashboard": "DM-Professional-Clinic", "title": "binlog event size", "group": "relay log", "used_for": "Used for monitoring the size distribution of binlog events written to relay logs in DM (Data Migration), showing p90, p95, and p99 percentiles. Helps identify abnormally large binlog events that could impact relay log performance and diagnose replication throughput issues.", "component": "", "phenomenon": "", "object": "", "percentile": [ 90, 95, 99 ], "variables": [ "instance", "job_id" ], "promql": [ "histogram_quantile(0.90, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.99, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "dm.json" }, { "panel_id": 81, "dashboard": "DM-Professional-Clinic", "title": "workers' state", "group": "HA", "used_for": "Used for monitoring the operational state of DM (Data Migration) workers in the cluster. Essential for tracking worker availability and diagnosing high availability issues when workers become unavailable or fail over.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (worker) (dm_master_worker_state)" ], "dashboard_file": "dm.json" }, { "panel_id": 125, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB seek operation latency in TiFlash-Proxy, tracking maximum, P99, and P95 durations. Critical for diagnosing read performance issues when TiFlash queries need to locate specific keys in the storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tiflash_proxy_tikv_engine_seek_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 82, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Get duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB point lookup (Get operation) latency in TiFlash-Proxy, showing maximum, P99, and P95 durations in microseconds. Essential for diagnosing read performance issues and identifying slow Get operations that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tiflash_proxy_tikv_engine_get_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 158, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page write Duration", "group": "PageStorage", "used_for": "Used for monitoring write latency percentiles (P80, P95, P99, P999) in TiFlash's PageStorage engine. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow page write operations that could impact columnar data ingestion and storage efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99, 999 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 129, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Seek operations", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring RocksDB seek operation rates in TiFlash-Proxy, including total seeks, successful seeks, and next operations. Essential for diagnosing read performance issues and understanding data access patterns in TiFlash's storage engine, particularly when investigating slow queries or high I/O overhead.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_locate{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 85, "dashboard": "DM-Professional-Clinic", "title": "number of worker event error", "group": "HA", "used_for": "Used for monitoring the count of worker event errors in DM (Data Migration) Master, categorized by error type. Critical for diagnosing high availability issues and identifying specific failure patterns in DM worker event processing that may impact data migration reliability.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (type) (dm_master_worker_event_error)" ], "dashboard_file": "dm.json" }, { "panel_id": 129, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage File Num", "group": "PageStorage", "used_for": "Used for monitoring the number of blob files and log files in TiFlash's PageStorage engine per instance. Essential for tracking storage file accumulation and identifying potential issues with file compaction or cleanup processes that could impact storage efficiency and performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_BlobFileNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_asynchronous_metric_LogNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 163, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Tasks OPM", "group": "PageStorage", "used_for": "Used for monitoring the rate of PageStorage garbage collection tasks per minute in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential issues with storage space reclamation or excessive GC overhead affecting TiFlash performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 82, "dashboard": "DM-Professional-Clinic", "title": "shard ddl error per minute", "group": "HA", "used_for": "Used for monitoring the rate of shard DDL errors per minute in DM (Data Migration) master, broken down by task and error type. Critical for identifying DDL synchronization failures in sharded table migration scenarios and diagnosing high availability issues in DM clusters.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (increase(dm_master_shard_ddl_error[1m]))" ], "dashboard_file": "dm.json" }, { "panel_id": 162, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Page GC Duration", "group": "PageStorage", "used_for": "Used for monitoring the 99th percentile latency of TiFlash PageStorage garbage collection operations. Essential for diagnosing performance degradation in TiFlash's storage layer, particularly when GC pauses impact query response times or cause storage management bottlenecks.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 164, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Numer of Pages", "group": "PageStorage", "used_for": "Used for monitoring the number of pages currently held in memory by TiFlash's PageStorage engine. Essential for tracking memory usage patterns and diagnosing potential memory pressure or page cache issues in TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_PagesInMem{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 84, "dashboard": "DM-Professional-Clinic", "title": "number of pending shard ddl", "group": "HA", "used_for": "**Used For:**\n\nUsed for monitoring the count of pending DDL operations in DM (Data Migration) shard merge scenarios, grouped by task and type. Critical for identifying DDL synchronization bottlenecks and ensuring coordinated schema changes across sharded tables are progressing normally in high availability deployments.", "component": "", "phenomenon": "", "object": "", "percentile": [], "variables": [], "promql": [ "sum by (task, type) (dm_master_ddl_state_number)" ], "dashboard_file": "dm.json" }, { "panel_id": 139, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of write operations in TiFlash-Proxy's RocksDB storage engine, including successful writes (self/other), write timeouts, and WAL writes. Essential for diagnosing TiFlash write performance issues, identifying write bottlenecks, and detecting timeout anomalies that may indicate storage layer problems.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_write_served{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 137, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of Write-Ahead Log (WAL) sync operations in TiFlash-Proxy's RocksDB engine. Critical for diagnosing write durability overhead and identifying potential I/O bottlenecks when WAL sync frequency becomes excessive or irregular.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 69, "dashboard": "DM-Professional-Clinic", "title": "auto-resume error", "group": "operate error", "used_for": "Used for monitoring auto-resume operation failures in DM (Data Migration) workers. Critical for identifying when automatic task recovery mechanisms fail, requiring manual intervention to restore data synchronization pipelines.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"AutoResume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 71, "dashboard": "DM-Professional-Clinic", "title": "before any operate error", "group": "operate error", "used_for": "Used for monitoring errors that occur in DM (Data Migration) workers before any operation begins. Helps diagnose pre-operation validation failures, connection issues, or configuration problems that prevent DM tasks from starting properly.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"BeforeAnyOp\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 231, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage Pending Writers Num", "group": "PageStorage", "used_for": "Used for monitoring the number of pending write operations in TiFlash's PageStorage component. Helps identify write bottlenecks and I/O congestion when the pending writer queue builds up, indicating potential disk performance issues or write pressure.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_PSPendingWriterNum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 76, "dashboard": "DM-Professional-Clinic", "title": "start error", "group": "operate error", "used_for": "Used for monitoring start operation errors in DM (Data Migration) workers. Essential for diagnosing failures when DM tasks fail to start, helping identify configuration issues, resource constraints, or connectivity problems that prevent data migration tasks from initializing properly.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Start\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 135, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "WAL sync duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) file synchronization operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing write performance bottlenecks and disk I/O issues, as high WAL sync duration directly impacts transaction commit latency and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 130, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write WAL duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of Write-Ahead Log (WAL) operations in TiFlash-Proxy's RocksDB engine, tracking maximum, P99, and P95 durations. Critical for diagnosing write performance bottlenecks and disk I/O issues in TiFlash's storage layer, as high WAL write latency directly impacts data durability and overall write throughput.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 126, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write operation latency in TiFlash-Proxy's RocksDB storage engine, showing maximum, 99th percentile, and 95th percentile write durations. Essential for diagnosing TiFlash write performance bottlenecks and identifying when storage layer operations are causing delays in data synchronization from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 133, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage WriteBatch Size", "group": "PageStorage", "used_for": "Used for monitoring the distribution of write batch sizes in TiFlash's PageStorage v3 engine. Helps diagnose write efficiency and identify whether data is being written in optimal batch sizes, which is critical for understanding I/O patterns and potential performance bottlenecks in TiFlash storage operations.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_page_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"v3\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 74, "dashboard": "DM-Professional-Clinic", "title": "pause error", "group": "operate error", "used_for": "Used for monitoring the number of pause operation errors in DM (Data Migration) workers. Helps diagnose issues when attempting to pause data migration tasks, indicating potential problems with task control or worker state management.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Pause\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 123, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Number of Tables", "group": "PageStorage", "used_for": "Used for monitoring the number of tables stored in TiFlash's PageStorage across different storage pool modes (V2, V3, and mixed mode). Helps administrators track table distribution across TiFlash instances and identify storage mode transitions or imbalances in table placement.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 78, "dashboard": "DM-Professional-Clinic", "title": "resume error", "group": "operate error", "used_for": "Used for monitoring the error rate of resume operations in DM (Data Migration) workers. Critical for identifying failures when attempting to resume paused or stopped data migration tasks, helping diagnose task recovery issues.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Resume\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 232, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PS Command OPS By Instance", "group": "PageStorage", "used_for": "Used for monitoring the operations per second (OPS) of PageStorage commands in TiFlash, broken down by instance and command type. Essential for identifying which TiFlash instances are handling the most PageStorage operations and detecting imbalances or performance bottlenecks in the storage layer.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_page_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 35, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Read Index OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft read index operations per second in TiFlash instances. Helps identify read throughput patterns and diagnose issues when TiFlash queries require strong consistency reads through the Raft protocol.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 198, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "PageStorage stored bytes by type", "group": "PageStorage", "used_for": "Used for monitoring the distribution of stored data bytes across different types in TiFlash's PageStorage engine. Helps identify storage space usage patterns and detect potential storage imbalances or abnormal growth in specific data types.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_page_data_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 167, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Stale Read OPS", "group": "Raft", "used_for": "Used for monitoring the rate of stale read operations per second on each TiFlash instance. Helps track the usage of stale read feature and identify instances handling high volumes of stale read requests, which is useful for understanding read workload patterns and optimizing consistency-latency tradeoffs.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_stale_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 75, "dashboard": "DM-Professional-Clinic", "title": "stop error", "group": "operate error", "used_for": "Used for monitoring the frequency of stop operation errors in DM (Data Migration) workers. Critical for identifying issues when attempting to stop data migration tasks, helping diagnose worker management and task lifecycle problems.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Stop\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 102, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache size", "group": "RocksDB - $db", "used_for": "Used for monitoring the top 20 block cache sizes across different column families in TiFlash-Proxy's RocksDB engine. Essential for diagnosing memory usage patterns and identifying potential cache pressure issues that could impact read performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 103, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable size", "group": "RocksDB - $db", "used_for": "Used for monitoring the memory consumption of RocksDB memtables in TiFlash-Proxy, broken down by column family. Essential for diagnosing memory pressure issues and understanding write buffer usage that can impact write performance and memory stability.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 128, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of RocksDB compaction operations in TiFlash-Proxy, broken down by operation type. Essential for diagnosing storage layer performance issues, identifying compaction bottlenecks, and understanding write amplification patterns that may impact TiFlash query performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_event_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 72, "dashboard": "DM-Professional-Clinic", "title": "source bound error", "group": "operate error", "used_for": "Used for monitoring source binding operation errors in DM (Data Migration) workers. Critical for diagnosing connection failures or configuration issues when DM workers attempt to bind to upstream MySQL/MariaDB data sources, helping identify source connectivity problems that block data replication.", "component": "", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"SourceBound\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 88, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Memtable hit", "group": "RocksDB - $db", "used_for": "Used for monitoring the memtable hit rate in TiFlash-Proxy's RocksDB storage engine. Helps diagnose memory efficiency and read performance by showing how often data is found in memory (memtable) versus requiring disk access, with higher hit rates indicating better memory utilization.", "component": "TIKV", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"memtable_miss\"}[2m])))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 76, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (decode)", "group": "Raft", "used_for": "Used for monitoring the time spent decoding Raft write data in TiFlash regions. Essential for diagnosing TiFlash write performance issues, particularly when investigating whether data decoding is causing bottlenecks in the Raft replication process.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 242, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Heavy Raft Apply Duration", "group": "Raft", "used_for": "Used for monitoring the 99th percentile latency of Raft command apply operations in TiFlash. Critical for diagnosing replication lag and data synchronization performance issues between TiKV and TiFlash, especially when queries show stale data or replication falls behind.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 77, "dashboard": "DM-Professional-Clinic", "title": "update error", "group": "operate error", "used_for": "Used for monitoring the rate of update operation errors in DM (Data Migration) workers. Helps identify issues with metadata updates or configuration changes that may be failing, indicating potential problems with DM's operational stability or connectivity to PD.", "component": "PD", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [], "promql": [ "dm_worker_operate_error{type=\"Update\"}" ], "dashboard_file": "dm.json" }, { "panel_id": 140, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "SST read duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the latency of SST (Sorted String Table) file read operations in TiFlash-Proxy's RocksDB storage engine, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage layer performance issues, particularly when queries experience slow data retrieval from disk or when investigating I/O bottlenecks in the columnar storage engine.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tiflash_proxy_tikv_engine_sst_read_micros{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 114, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog file position being processed by the DM (Data Migration) validator component. Helps track data validation progress and identify potential lag or stalls in the validation process when troubleshooting data migration issues.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_file" ], "dashboard_file": "dm.json" }, { "panel_id": 136, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction duration", "group": "RocksDB - $db", "used_for": "Used for monitoring RocksDB compaction operation latency in TiFlash-Proxy, showing maximum, P99, and P95 durations. Critical for diagnosing TiFlash storage performance issues, as prolonged compaction times can indicate write amplification problems or resource bottlenecks affecting query performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tiflash_proxy_tikv_engine_compaction_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 118, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in file)", "group": "Validator Statistics", "used_for": "Used for monitoring the latency of validator binlog processing from file in DM (Data Migration). Helps diagnose delays in data validation when reading and processing binlog events from local files, which is critical for identifying bottlenecks in the validation pipeline.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logfile_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 37, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Wait Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash waiting for Raft log index synchronization at p80, p95, and p99 percentiles. Critical for diagnosing data replication delays between TiKV and TiFlash, helping identify when TiFlash replicas fall behind in applying Raft logs, which can impact query freshness and read performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 116, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Latency (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the data replication lag in bytes between the DM validator and the source binlog position. Critical for identifying data synchronization delays and ensuring the validator is keeping pace with upstream changes during data migration validation.", "component": "", "phenomenon": "latency", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_logpos_latency" ], "dashboard_file": "dm.json" }, { "panel_id": 87, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write stall duration", "group": "RocksDB - $db", "used_for": "Used for monitoring write stall duration in TiFlash-Proxy's RocksDB engine, tracking maximum, P99, and P95 latencies when writes are blocked due to compaction pressure or memtable limits. Critical for diagnosing write performance degradation and identifying when TiFlash-Proxy experiences backpressure that could impact data replication from TiKV.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tiflash_proxy_tikv_engine_write_stall{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 82, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration", "group": "Raft", "used_for": "Used for monitoring the p99 latency of applying Raft write logs in TiFlash's storage engine. Critical for diagnosing write performance bottlenecks and replication lag issues when TiFlash processes data changes from TiKV through the Raft protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) ", "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m]))", "sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m]) ) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 80, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache hit", "group": "RocksDB - $db", "used_for": "**Used For:**\n\nUsed for monitoring the block cache hit ratio in TiFlash-Proxy's RocksDB storage engine, including overall cache hits, data block hits, and filter block hits. Essential for diagnosing read performance issues and determining if the block cache size is sufficient to avoid excessive disk I/O.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[2m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[2m])))", "sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[2m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 120, "dashboard": "DM-Professional-Clinic", "title": "Validator Error Count", "group": "Validator Statistics", "used_for": "Used for monitoring the total number of validation errors encountered by DM's data validator component. Essential for identifying data consistency issues during migration and quickly detecting when source and target data diverge or validation checks fail.", "component": "", "phenomenon": "error_rate", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_error_count" ], "dashboard_file": "dm.json" }, { "panel_id": 36, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Batch Read Index Duration", "group": "Raft", "used_for": "Used for monitoring the latency of batch read index operations in TiFlash's Raft layer at different percentiles (p80, p95, p99). Critical for diagnosing TiFlash read performance issues and identifying delays in obtaining consistent read snapshots from the Raft consensus protocol.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 467, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read/write traffic flow and data insertion rate of TiFlash-Proxy's RocksDB block cache. Essential for analyzing cache utilization patterns and diagnosing memory pressure or cache efficiency issues in TiFlash storage engine.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 468, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Block cache operations", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of block cache insertion operations in TiFlash-Proxy's RocksDB storage engine, including overall block additions, data block additions, and filter block additions. Essential for understanding cache population patterns and diagnosing issues related to block cache efficiency and memory pressure in TiFlash storage nodes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 131, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Total keys", "group": "RocksDB - $db", "used_for": "Used for monitoring the estimated total number of keys stored in each column family of TiFlash-Proxy's RocksDB storage engine. Helps assess data distribution across column families and identify potential storage imbalances or unexpected data growth patterns.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_engine_estimate_num_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 3, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring TiProxy server CPU utilization and comparing it against the maximum available CPU cores (maxprocs). Essential for identifying CPU bottlenecks, resource saturation, and determining if TiProxy instances need vertical scaling or load redistribution.", "component": "TIPROXY", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}[2m])", "tiproxy_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 6, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiProxy instances. Essential for tracking connection pool utilization, identifying connection leaks, and diagnosing capacity issues when connection counts approach limits or show unexpected spikes.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tiproxy_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 5, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiProxy instance has been running since its last restart. Essential for tracking instance stability, identifying unexpected restarts, and verifying deployment or maintenance activities.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 132, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Keys flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the rate of key operations in TiFlash-Proxy's RocksDB storage engine, including keys read, keys written, and corrupted keys detected during compaction. Essential for diagnosing data integrity issues and understanding the read/write workload patterns at the key level in TiFlash storage.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 93, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft admin logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft administrative commands in TiFlash through a heatmap visualization. Essential for diagnosing performance issues related to schema changes, region splits, and other administrative operations that affect TiFlash's Raft replication layer.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiProxy's memory consumption, including total resident memory and Go heap usage. Essential for detecting memory leaks, identifying out-of-memory risks, and ensuring TiProxy instances have adequate memory resources for connection pooling and query routing operations.", "component": "TIPROXY", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 81, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Raft write logs Duration [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the latency distribution of applying Raft write log commands in TiFlash nodes. Essential for diagnosing write replication delays and identifying performance bottlenecks in TiFlash's Raft log application process, particularly when investigating slow data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 85, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the read throughput of TiFlash-Proxy's RocksDB storage engine, tracking both direct bytes read and iterator-based reads. Essential for diagnosing TiFlash read performance issues, identifying I/O bottlenecks, and understanding data access patterns during analytical query execution.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 9, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Goroutine Count", "group": "Server", "used_for": "Used for monitoring the number of active goroutines in TiProxy instances. Helps identify potential goroutine leaks, resource exhaustion, or abnormal concurrency patterns that could impact proxy performance and stability.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tiproxy\"}", "sum(go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiproxy\"})" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 87, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region write Duration (write blocks)", "group": "Raft", "used_for": "Used for monitoring the latency distribution of TiFlash region write operations when persisting data blocks to storage. Critical for diagnosing TiFlash write performance bottlenecks and identifying slow storage I/O issues that impact data replication from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 243, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Entry Batch Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft log entry batch sizes in TiFlash across different size buckets. Helps identify abnormal batch size patterns that could indicate replication inefficiencies or network issues between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_entry_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"normal\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 235, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft consensus events per second in TiFlash, broken down by event type. Essential for diagnosing Raft replication issues, identifying abnormal consensus activity, and troubleshooting data synchronization problems between TiFlash and TiKV.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 241, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Frequent Events QPS", "group": "Raft", "used_for": "Used for monitoring the rate of frequent Raft protocol events in TiFlash, broken down by event type. Essential for diagnosing Raft replication activity, identifying abnormal consensus operations, and troubleshooting data synchronization issues between TiFlash replicas.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_raft_frequent_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 8, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Disconnection OPM", "group": "Server", "used_for": "Used for monitoring the rate of client disconnections per minute in TiProxy, categorized by disconnection type. Essential for identifying connection stability issues, detecting abnormal disconnect patterns, and diagnosing network or client-side problems affecting database connectivity.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 86, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write flow", "group": "RocksDB - $db", "used_for": "Used for monitoring write throughput in TiFlash-Proxy's RocksDB storage engine, tracking both WAL (Write-Ahead Log) file writes and total bytes written. Essential for diagnosing write performance bottlenecks, identifying write amplification issues, and understanding I/O patterns in TiFlash's storage layer.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 133, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Read", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data read per RocksDB read operation in TiFlash-Proxy, showing maximum, P99, and P95 values. Helps diagnose read amplification issues and identify whether queries are reading excessive data, which can indicate inefficient storage layout or suboptimal query patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_read{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 134, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Bytes / Write", "group": "RocksDB - $db", "used_for": "Used for monitoring the amount of data written per write operation in TiFlash-Proxy's RocksDB engine, showing maximum, P99, and P95 values. Helps diagnose write amplification issues and identify whether write operations are batching efficiently or performing many small writes that could impact I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "max(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tiflash_proxy_tikv_engine_bytes_per_write{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 234, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Log Gap Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft log gaps between applied_index and compact_index in TiFlash replicas as a heatmap. Essential for identifying replication lag patterns and diagnosing issues where TiFlash falls behind in applying or compacting Raft logs, which can impact data freshness and query performance.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"applied_index\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"compact_index\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 7, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Create Connection OPM", "group": "Server", "used_for": "Used for monitoring the rate of new connection creation per minute on TiProxy instances. Helps identify connection churn patterns, detect connection storms, and diagnose issues with client connection behavior or connection pooling inefficiencies.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_server_create_connection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 12, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency through TiProxy at the 95th and 99th percentiles. Essential for identifying performance degradation in client-to-TiDB connections and diagnosing whether slowdowns originate from the proxy layer or backend TiDB servers.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 90, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction flow", "group": "RocksDB - $db", "used_for": "Used for monitoring the data flow rates during RocksDB compaction operations in TiFlash-Proxy, including bytes read, bytes written during compaction, and flush write operations. Essential for diagnosing I/O bottlenecks, understanding storage engine write amplification, and identifying excessive compaction activity that may impact TiFlash performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[2m]))", "sum(rate(tiflash_proxy_tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 14, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Backend", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency from TiProxy to each backend TiDB instance. Essential for identifying which specific backend servers are experiencing slow query performance and diagnosing uneven load distribution or backend-specific issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, backend)), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 863, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compression ratio", "group": "RocksDB - $db", "used_for": "Used for monitoring the compression efficiency of data at different LSM-tree levels in TiFlash-Proxy's RocksDB storage engine. Helps identify compression effectiveness and potential storage optimization opportunities, with lower ratios indicating better compression and reduced disk space usage.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 516, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number of snapshots", "group": "RocksDB - $db", "used_for": "Used for monitoring the number of active RocksDB snapshots in TiFlash-Proxy's storage engine. Helps identify potential memory pressure or snapshot accumulation issues that could impact read performance and resource utilization.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_num_snapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 112, "dashboard": "DM-Professional-Clinic", "title": "Validator Binlog Position (in byte)", "group": "Validator Statistics", "used_for": "Used for monitoring the current binlog position (in bytes) being processed by the DM validator component. Helps track data validation progress and identify potential lag or stalls in the validation process during data migration tasks.", "component": "", "phenomenon": "io", "object": "", "percentile": [], "variables": [], "promql": [ "dm_validator_validator_binlog_pos" ], "dashboard_file": "dm.json" }, { "panel_id": 127, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction pending bytes", "group": "RocksDB - $db", "used_for": "Used for monitoring the volume of data pending compaction in TiFlash-Proxy's RocksDB storage engine, broken down by column family. Critical for identifying compaction backlogs that can lead to read amplification, increased disk usage, and degraded I/O performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 13, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "P99 Duration By Instance", "group": "Query Summary", "used_for": "Used for monitoring the 99th percentile query latency for each TiProxy instance. Essential for identifying performance degradation, comparing latency across instances, and diagnosing which specific TiProxy nodes are experiencing slow query issues.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, instance))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 75, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Applying snapshots Count", "group": "Raft", "used_for": "Used for monitoring the number of Raft snapshots currently being applied and prehandled in TiFlash instances. Critical for diagnosing region replication delays, identifying snapshot application bottlenecks, and troubleshooting data synchronization issues between TiKV and TiFlash.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_RaftNumPrehandlingSubTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 240, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC OPS", "group": "Raft", "used_for": "Used for monitoring the rate of Raft eager garbage collection operations in TiFlash, broken down by GC type. Helps identify GC activity patterns and potential issues with Raft log cleanup that could impact memory usage or replication performance.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_eager_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 518, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read amplication", "group": "RocksDB - $db", "used_for": "Used for monitoring the read amplification ratio in TiFlash-Proxy's RocksDB storage engine, which measures how many bytes are actually read from disk versus bytes returned to queries. Critical for identifying inefficient data access patterns and LSM-tree compaction issues that cause excessive disk I/O and degrade read performance.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[2m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 244, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of region sizes in TiFlash, showing both flushed and unflushed data volumes across different size buckets. Helps identify region size imbalances, detect abnormally large regions that may impact I/O performance, and monitor the effectiveness of TiFlash's data flushing mechanism.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"unflushed\"}[2m])) by (le, type)", "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"flushed\"}[2m])) by (le, type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 127, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode SST to DT Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding SST files to DeltaTree format during Raft snapshot application. Critical for diagnosing slow snapshot ingestion and identifying performance bottlenecks when TiFlash replicas are catching up with data changes from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode_sst2dt\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 249, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ongoing raft snapshot", "group": "Raft", "used_for": "Used for monitoring the rate of ongoing Raft snapshot data transfer in TiFlash, broken down by snapshot type (apply/generate). Essential for diagnosing region replication issues, identifying snapshot bottlenecks, and understanding data synchronization overhead between TiFlash nodes.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "__rate_interval", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 15, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Instance", "group": "Query Summary", "used_for": "Used for monitoring the number of commands per second (CPS) processed by each TiProxy instance. Helps identify load distribution across TiProxy instances and detect performance bottlenecks or imbalanced traffic routing.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 74, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Ingest SST Duration", "group": "Raft", "used_for": "Used for monitoring the latency distribution of SST file ingestion operations in TiFlash's Raft layer. Critical for diagnosing slow data replication from TiKV and identifying performance bottlenecks during snapshot application or region data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 239, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Eager GC Duration", "group": "Raft", "used_for": "Used for monitoring the latency of TiFlash's Raft eager garbage collection operations at p95 and p99 percentiles. Essential for diagnosing performance issues related to Raft log cleanup and identifying when eager GC operations are taking longer than expected, which could impact TiFlash's replication efficiency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "sum(rate(tiflash_raft_eager_gc_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_raft_eager_gc_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2003, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Ingest SST duration seconds", "group": "RocksDB - $db", "used_for": "Used for monitoring the 99th percentile latency of SST file ingestion operations in TiFlash-Proxy's RocksDB storage engine. Critical for diagnosing snapshot application performance issues and identifying bottlenecks during data replication from TiKV to TiFlash.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (le))", "sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 17, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) in TiProxy, broken down by command type (SELECT, INSERT, UPDATE, etc.). Helps identify query workload distribution and detect unusual command patterns or sudden changes in specific query types.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (cmd_type)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 517, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Oldest snapshots duration", "group": "RocksDB - $db", "used_for": "Used for monitoring the duration of the oldest active snapshot in TiFlash-Proxy's RocksDB engine. Critical for identifying long-running transactions or stale snapshots that can block compaction, cause storage bloat, and degrade read/write performance.", "component": "TIKV", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_oldest_snapshot_duration{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 250, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Size Heatmap", "group": "Raft", "used_for": "Used for visualizing the distribution of Raft snapshot sizes in TiFlash over time as a heatmap. Helps identify abnormal snapshot size patterns, detect large snapshot transfers that may impact network bandwidth, and diagnose replication performance issues in TiFlash's Raft layer.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_snapshot_total_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"approx_raft_snapshot\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 72, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Predecode Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash spends pre-decoding Raft snapshots during data replication. Critical for diagnosing slow snapshot application issues and identifying bottlenecks in TiFlash's Raft log replay process that could delay data synchronization from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 20, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Backend Connections", "group": "Balance", "used_for": "Used for monitoring the number of active connections between TiProxy and backend TiDB instances. Essential for tracking connection distribution across TiDB nodes and diagnosing connection balancing issues or backend connectivity problems.", "component": "TIPROXY", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_balance_b_conn{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 2381, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Stall conditions changed of each CF", "group": "RocksDB - $db", "used_for": "Used for monitoring the frequency of stall condition changes in TiFlash-Proxy's RocksDB column families. Helps diagnose write throttling and backpressure issues when RocksDB triggers stalls due to memtable limits, L0 file accumulation, or pending compaction bytes.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_engine_stall_conditions_changed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 2452, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Write Stall Reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the reasons causing write stalls in TiFlash-Proxy's RocksDB storage engine. Essential for diagnosing write performance degradation by identifying specific bottlenecks such as memtable limits, L0 file accumulation, or pending compaction issues.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 16, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "CPS by Backend", "group": "Query Summary", "used_for": "Used for monitoring the number of queries per second (CPS) handled by each TiDB backend instance through TiProxy. Helps identify load distribution imbalances across backend servers and detect if specific TiDB instances are receiving disproportionate query traffic.", "component": "TIPROXY", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_session_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 71, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Keys flow", "group": "Raft", "used_for": "Used for monitoring the rate of Raft key operations in TiFlash, broken down by operation type (such as apply, snapshot, etc.). Essential for understanding TiFlash's data replication throughput and diagnosing issues with Raft log application or data synchronization delays.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4453, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption data keys", "group": "Encryption", "used_for": "Used for monitoring the total number of encryption data keys stored in TiFlash-Proxy's encryption key management system. Essential for tracking encryption key lifecycle and diagnosing issues with data-at-rest encryption configuration or key rotation problems.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_data_key_storage_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 18, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Handshake Duration", "group": "Query Summary", "used_for": "Used for monitoring the 95th and 99th percentile latency of client connection handshakes through TiProxy. Critical for diagnosing slow connection establishment issues and identifying authentication or SSL handshake bottlenecks that impact client connection times.", "component": "TIPROXY", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_session_handshake_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_session_handshake_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_session_handshake_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 21, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration OPM", "group": "Balance", "used_for": "Used for monitoring the rate of session migrations per minute in TiProxy's load balancing operations. Essential for tracking how frequently TiProxy moves client connections between TiDB instances and diagnosing connection rebalancing behavior or potential migration issues.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(label_replace(sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (migrate_res, from, to), \"from\", \"$1\", \"from\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\"), \"to\", \"$1\", \"to\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 73, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Snapshot Flush Duration", "group": "Raft", "used_for": "Used for monitoring the time TiFlash takes to flush Raft snapshots to disk. Critical for diagnosing slow snapshot application issues that can cause TiFlash replicas to lag behind TiKV, impacting data freshness and query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4492, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encrypted files", "group": "Encryption", "used_for": "Used for monitoring the number of encrypted files in TiFlash-Proxy's storage layer. Helps verify encryption-at-rest is functioning correctly and track the volume of encrypted data files across the cluster.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_file_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 92, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency [Heatmap]", "group": "Raft", "used_for": "Used for monitoring the distribution of Raft log replication latency from TiKV to TiFlash instances over time. Essential for diagnosing data synchronization delays and identifying network or replication bottlenecks between TiKV and TiFlash nodes.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 2002, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Number files at each level", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of SST files across different levels in TiFlash-Proxy's RocksDB storage engine for each column family. Essential for diagnosing compaction issues, identifying level imbalances, and understanding storage layer health that could impact read/write performance.", "component": "TIKV", "phenomenon": "", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg(tiflash_proxy_tikv_engine_num_files_at_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 4496, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption initialized", "group": "Encryption", "used_for": "Used for verifying whether encryption-at-rest has been successfully initialized in TiFlash-Proxy instances. Essential for confirming encryption setup during cluster deployment or when troubleshooting encryption-related configuration issues.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_is_initialized{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 22, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Duration", "group": "Balance", "used_for": "Used for monitoring the latency of session migration operations in TiProxy during load balancing, showing p95 and p99 percentiles. Critical for diagnosing performance issues when TiProxy redistributes client connections across TiDB instances, helping identify slow migrations that could impact user experience during rebalancing events.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_balance_migrate_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_balance_migrate_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_balance_migrate_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 25, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Get Backend Duration", "group": "Backend", "used_for": "Used for monitoring the latency of TiProxy selecting and connecting to backend TiDB servers, showing p95, p99, and average durations. Critical for diagnosing connection establishment delays and backend selection performance issues that could impact client connection times.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiproxy_backend_get_backend_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiproxy_backend_get_backend_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiproxy_backend_get_backend_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 26, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Ping Backend Duration", "group": "Backend", "used_for": "Used for monitoring the network latency between TiProxy and backend TiDB servers through health check pings. Essential for diagnosing connection quality issues and identifying network problems that could affect query routing performance.", "component": "TIPROXY", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(tiproxy_backend_ping_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}, \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 2451, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Compaction reason", "group": "RocksDB - $db", "used_for": "Used for monitoring the distribution of RocksDB compaction triggers in TiFlash-Proxy by column family and reason type. Helps diagnose storage layer performance issues by identifying whether compactions are triggered by size thresholds, TTL, manual operations, or other factors, which is critical for understanding write amplification and I/O patterns.", "component": "TIKV", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "db", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_proxy_tikv_engine_compaction_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", db=\"$db\"}[2m])) by (cf, reason)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 29, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of data received by TiProxy from backend TiDB servers, measured in bytes per second. Helps identify backend-to-proxy traffic patterns, detect network bottlenecks, and diagnose imbalanced data distribution across TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 4497, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Encryption meta files size", "group": "Encryption", "used_for": "Used for monitoring the size of encryption metadata files in TiFlash-Proxy. Helps track storage overhead from encryption key management and identify potential issues with metadata file growth that could impact disk usage.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_proxy_tikv_encryption_meta_file_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 97, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate Histogram", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the distribution of rough set filter effectiveness in TiFlash's storage layer. Helps identify how efficiently TiFlash is filtering data blocks during queries, with higher filter rates indicating better query performance through reduced data scanning.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 91, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Upstream Latency", "group": "Raft", "used_for": "Used for monitoring the latency of Raft log replication from TiKV to TiFlash at p95, p99, and maximum percentiles. Critical for diagnosing data synchronization delays between TiKV and TiFlash, helping identify network issues or upstream bottlenecks that impact TiFlash's real-time data freshness.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))", "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) " ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 27, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Health Check Cycle", "group": "Backend", "used_for": "Used for monitoring the health check cycle duration of TiProxy backend connections to TiDB instances. Essential for diagnosing connection pool health issues and detecting when backend health checks are taking longer than expected, which may indicate network problems or TiDB instance unavailability.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_backend_health_check_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 23, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Session Migration Reasons", "group": "Balance", "used_for": "Used for tracking the reasons why TiProxy migrates client sessions between TiDB instances during load balancing operations. Helps diagnose balance behavior and identify whether migrations are triggered by planned rebalancing, instance failures, or configuration changes.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tiproxy_balance_migrate_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (reason)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 10, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Owner", "group": "Server", "used_for": "Used for identifying which TiProxy instance is currently serving as the cluster owner/leader. Essential for troubleshooting high availability scenarios, understanding leadership changes, and diagnosing coordination issues in multi-instance TiProxy deployments.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiproxy_server_owner{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 173, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Read Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency breakdown of read operations in TiFlash's disaggregated storage architecture. Helps identify which stage of the disaggregated read path (e.g., remote fetch, local cache, network transfer) is contributing most to query latency, essential for diagnosing performance bottlenecks in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4498, "dashboard": "TiFlash-Proxy-Details-Cluster-Clinic", "title": "Read/write encryption meta duration", "group": "Encryption", "used_for": "Used for monitoring the latency of reading and writing encryption metadata files in TiFlash-Proxy, showing p95 and maximum durations. Critical for diagnosing performance issues with encryption-at-rest operations and identifying when metadata I/O becomes a bottleneck affecting TiFlash query performance.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 95 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation))", "sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, operation)" ], "dashboard_file": "tiflash-proxy.json" }, { "panel_id": 30, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second from Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets received by TiProxy from backend TiDB servers. Helps identify network throughput patterns, backend communication issues, and potential packet loss or network bottlenecks between TiProxy and TiDB instances.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_inbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 3, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiDB server instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and whether servers have been unexpectedly restarted due to crashes or maintenance operations.", "component": "TIDB", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 5, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring the CPU utilization rate of TiDB server instances. Essential for identifying CPU bottlenecks, resource contention, and determining if TiDB servers need vertical scaling or if query workloads need optimization.", "component": "TIDB", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}[2m])" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 187, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload Duration", "group": "Disaggregated", "used_for": "Used for monitoring the latency of checkpoint upload operations in TiFlash disaggregated storage architecture at different percentiles (p80, p90, p99). Critical for diagnosing slow checkpoint synchronization issues that could impact data durability and recovery time in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 174, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow", "group": "Disaggregated", "used_for": "Used for monitoring the data upload flow rate to remote storage in TiFlash disaggregated architecture, tracking both incremental checkpoint uploads and compaction-related uploads. Essential for diagnosing write-to-S3 performance issues and understanding checkpoint synchronization patterns in compute-storage separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"incremental\"}[2m]))", "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"compaction\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 176, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote File Num", "group": "Disaggregated", "used_for": "Used for monitoring the number of remote storage files in TiFlash's disaggregated storage architecture. Helps track remote file accumulation and identify potential issues with file management or storage tiering in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"num_files\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 4, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Connection Count", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the number of active client connections to TiDB instances. Essential for tracking connection usage patterns, identifying connection leaks, and diagnosing issues related to connection pool exhaustion or unexpected connection spikes that may impact database availability.", "component": "TIDB", "phenomenon": "io", "object": "connection", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 196, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload keys speed by type (all)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint key uploads in TiFlash's disaggregated storage architecture, broken down by key type. Helps diagnose checkpoint performance issues and identify which key types are contributing most to checkpoint upload activity in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_keys_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 32, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Packets/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the rate of network packets sent from TiProxy instances to backend TiDB servers. Helps identify network traffic distribution patterns, detect connection issues to specific backends, and diagnose packet-level communication problems between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_packets{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 8, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Duration", "group": "Query Summary", "used_for": "Used for monitoring query latency at the 95th and 99th percentiles in TiDB, excluding internal queries. Essential for identifying performance degradation, detecting slow query patterns, and understanding tail latency that affects user experience during troubleshooting or performance analysis.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) by (le))", "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type!=\"internal\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", sql_type!=\"internal\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 54, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Available size", "group": "Server", "used_for": "Used for monitoring the available disk space on each TiFlash instance. Critical for preventing storage exhaustion issues and planning capacity expansion before TiFlash nodes run out of disk space.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 53, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Store size", "group": "Server", "used_for": "Used for monitoring the storage space consumed by each TiFlash instance, including both local and remote storage usage. Essential for tracking storage capacity trends, planning disk space allocation, and identifying instances approaching storage limits before they impact cluster operations.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"\"}) by (instance)", "sum(tiflash_system_current_metric_StoreSizeUsedRemote{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 197, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Checkpoint Upload flow by type (incremental+compaction)", "group": "Disaggregated", "used_for": "Used for monitoring the rate of checkpoint data upload flow in TiFlash's disaggregated storage architecture, broken down by operation type (incremental updates and compaction). Essential for diagnosing checkpoint upload performance, identifying bottlenecks in data persistence, and understanding the balance between incremental and compaction workloads in write-compute separated deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_checkpoint_flow_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 33, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Cross Location Bytes/Second", "group": "Traffic", "used_for": "Used for monitoring the rate of cross-location network traffic (bytes per second) handled by each TiProxy instance. Essential for identifying cross-datacenter or cross-availability-zone traffic patterns and diagnosing potential network bottlenecks or unexpected data transfer costs in distributed deployments.", "component": "TIPROXY", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiproxy_traffic_cross_location_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 11, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS By Instance", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) executed by each TiDB instance. Helps identify uneven query distribution across instances, detect instance-level performance bottlenecks, and diagnose load balancing issues in the cluster.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 175, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Store Usage", "group": "Disaggregated", "used_for": "Used for monitoring remote storage usage in disaggregated TiFlash deployments, tracking total size, valid data size, and storage efficiency ratio per instance. Essential for managing cloud storage costs and identifying storage bloat or inefficient space utilization in TiFlash's remote storage layer.", "component": "TIFLASH", "phenomenon": "", "object": "store", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"}) by (instance)", "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) by (instance)", "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"})) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 55, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Capacity size", "group": "Server", "used_for": "**Used For:**\n\nUsed for monitoring the total storage capacity configured for each TiFlash instance. Essential for capacity planning and identifying instances approaching storage limits before they impact cluster availability.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 200, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region", "group": "Server", "used_for": "Used for monitoring the total number of Raft regions stored on each TiFlash instance. Essential for tracking data distribution across TiFlash nodes and identifying region imbalance issues that could impact query performance and storage utilization.", "component": "TIFLASH", "phenomenon": "io", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 9, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Failed Query OPS", "group": "Query Summary", "used_for": "Used for monitoring the rate of failed queries per second in TiDB, broken down by error type and instance. Essential for quickly identifying query execution failures, detecting error spikes, and diagnosing issues like connection problems, syntax errors, or resource constraints affecting query success rates.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(increase(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type, instance)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 189, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Request QPS", "group": "Disaggregated", "used_for": "Used for monitoring the rate of remote object lock requests per second in TiFlash's disaggregated storage architecture, broken down by request type. Essential for diagnosing lock contention issues and understanding coordination overhead between compute and storage nodes in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "throughput", "object": "request", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_disaggregated_object_lock_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 191, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Object Lock Duration", "group": "Disaggregated", "used_for": "Used for monitoring the 99th percentile latency of remote object lock requests in TiFlash's disaggregated storage architecture. Critical for diagnosing performance bottlenecks in distributed object storage access and identifying lock contention issues that may impact query execution speed.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 31, "dashboard": "TiProxy-Summary-Cluster-Clinic", "title": "Bytes/Second to Backends", "group": "Traffic", "used_for": "Used for monitoring the outbound network traffic rate from TiProxy instances to backend TiDB servers. Helps identify network bandwidth usage patterns, detect traffic imbalances across backends, and diagnose connectivity or throughput issues between TiProxy and TiDB nodes.", "component": "TIPROXY", "phenomenon": "", "object": "", "percentile": [], "variables": [ "1", "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "label_replace(sum(rate(tiproxy_traffic_outbound_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, backend), \"backend\", \"$1\", \"backend\", \"(.+-tidb-[0-9]+).*peer.*.svc.*\")" ], "dashboard_file": "tiproxy-summary.json" }, { "panel_id": 6, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Memory Usage", "group": "Server", "used_for": "Used for monitoring TiDB server memory consumption, including total resident memory and Go heap allocation. Essential for detecting memory leaks, identifying OOM risks, and understanding memory pressure during high-load scenarios or complex query execution.", "component": "TIDB", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}", "go_memory_classes_heap_objects_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"} + go_memory_classes_heap_unused_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", component=\"tidb\"}" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 13, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "CPS by CMD", "group": "Query Summary", "used_for": "Used for monitoring commands per second (CPS) in TiDB, broken down by command type (SELECT, INSERT, UPDATE, DELETE, etc.). Helps identify query workload composition and detect unusual command patterns or sudden changes in specific query types.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 21, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Uptime", "group": "Server", "used_for": "Used for monitoring how long each TiFlash instance has been running since its last restart. Helps identify recent restarts, instance stability issues, and verify successful deployments or rolling updates.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Command Per Second", "group": "Query Summary", "used_for": "Used for monitoring the rate of SQL commands executed per second in TiDB, broken down by execution result (success/error). Essential for tracking overall query throughput, comparing current performance against historical baselines (1-day offset), and quickly identifying sudden drops in successful queries or spikes in failed commands.", "component": "TIDB", "phenomenon": "", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (result)", "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", result=\"OK\"}[2m] offset 1d))", "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 51, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "CPU Usage", "group": "Server", "used_for": "Used for monitoring CPU utilization of TiFlash instances, showing both actual CPU usage rate and total available logical CPU cores. Essential for identifying CPU bottlenecks, resource saturation issues, and determining if TiFlash nodes need vertical or horizontal scaling.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])", "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 12, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "QPS", "group": "Query Summary", "used_for": "Used for monitoring the overall queries per second (QPS) in TiDB, broken down by statement type (SELECT, INSERT, UPDATE, DELETE, etc.). Essential for tracking cluster workload patterns, identifying traffic spikes, and understanding the distribution of different query types during performance analysis or troubleshooting.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 233, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory Usage of Storage Tasks", "group": "Disaggregated", "used_for": "Used for monitoring memory consumption of storage-related tasks in TiFlash's disaggregated architecture, including query storage tasks, page fetching operations, and delta index cache. Essential for diagnosing memory pressure issues and optimizing resource allocation in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_DT_DeltaIndexCacheSize{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)", "sum(tiflash_system_current_metric_MemoryTrackingSharedColumnData{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 18, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Queries Using Plan Cache OPS", "group": "Query Detail", "used_for": "Used for monitoring the rate of queries utilizing TiDB's plan cache, broken down by cache type (hit/miss). Essential for evaluating plan cache effectiveness and identifying opportunities to improve query performance through better cache utilization.", "component": "TIDB", "phenomenon": "throughput", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 15, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Parse Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL parsing latency in TiDB. Helps identify performance bottlenecks in the query parsing phase, particularly when dealing with complex SQL statements or syntax issues that cause excessive parsing overhead.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 193, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Duration Breakdown", "group": "Disaggregated", "used_for": "Used for monitoring the P90 and P99 latency breakdown of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for diagnosing performance bottlenecks in S3-based storage cleanup processes and identifying which GC operation types are experiencing high latency.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [ 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 10, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Memory", "group": "Server", "used_for": "Used for monitoring TiFlash's memory allocation patterns through jemalloc metrics, including retained, mapped, and resident memory. Essential for diagnosing memory leaks, identifying excessive memory consumption, and understanding TiFlash's memory footprint during query processing and data storage operations.", "component": "TIFLASH", "phenomenon": "memory", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})", "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}", "sum(tiflash_system_current_metric_MemoryCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 178, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Bytes", "group": "S3", "used_for": "Used for monitoring S3 storage I/O throughput in TiFlash, tracking read/write bytes per second including DMFile operations. Essential for diagnosing S3 storage performance bottlenecks and understanding data transfer patterns when TiFlash uses S3 as the storage backend.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3WriteDMFileBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 195, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote GC Status", "group": "Disaggregated", "used_for": "Used for monitoring the status of remote garbage collection operations in TiFlash's disaggregated storage architecture. Essential for tracking S3-based storage cleanup activities and diagnosing issues with stale data removal in cloud-native TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(tiflash_storage_s3_gc_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance,type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 21, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Transaction Duration", "group": "Transaction", "used_for": "Used for monitoring transaction execution latency at different percentiles (p80, p95, p99) across TiDB instances. Essential for identifying transaction performance degradation, detecting slow transaction patterns, and diagnosing issues affecting overall database responsiveness during transactional workloads.", "component": "TIDB", "phenomenon": "latency", "object": "transaction", "percentile": [ 80, 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))", "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le, txn_mode))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 20, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "TPS", "group": "Transaction", "used_for": "**Used For:**\n\nUsed for monitoring transactions per second (TPS) in TiDB, broken down by transaction type and mode (optimistic/pessimistic). Essential for tracking transaction throughput, identifying workload patterns, and detecting sudden changes in transaction volume that may indicate application issues or load shifts.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tidb_session_transaction_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, txn_mode)" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 16, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Compile Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile SQL compilation latency in TiDB, measuring the time spent parsing and optimizing queries before execution. Essential for diagnosing slow query performance caused by complex SQL statements or inefficient query plan generation.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 201, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Request Unit", "group": "Disaggregated", "used_for": "Used for monitoring Request Unit (RU) consumption in TiFlash disaggregated storage and compute architecture. Helps track resource usage patterns across keyspaces and compute clusters, essential for capacity planning and cost analysis in serverless TiDB deployments.", "component": "TIFLASH", "phenomenon": "", "object": "request", "percentile": [], "variables": [], "promql": [ "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)", "sum(increase(tiflash_storage_sync_replica_ru[24h])) by (keyspace_id)", "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)", "sum(increase(tiflash_compute_request_unit[24h])) by (cluster_id)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 179, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operations (PUT, GET, HEAD) performed by TiFlash instances when using S3-compatible storage. Essential for diagnosing S3 storage throughput issues, identifying unusual access patterns, and troubleshooting performance problems related to remote storage operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 236, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "DeltaIndexCache", "group": "Disaggregated", "used_for": "Used for monitoring the delta index cache hit rate and access patterns in TiFlash's disaggregated storage architecture. Essential for evaluating cache efficiency and diagnosing performance issues related to delta index lookups, where low hit rates may indicate insufficient cache size or suboptimal data access patterns.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, instance)", "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit\"}[2m])) by (instance) /sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 145, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Task", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash region task threads, which handle region-related operations like region split, merge, and metadata updates. Essential for diagnosing performance bottlenecks in TiFlash's region management and identifying when region operations are consuming excessive CPU resources.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 182, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Retry OPS", "group": "S3", "used_for": "Used for monitoring the rate of S3 operation retries in TiFlash, including GET, PUT, and DMFile operations. Essential for identifying S3 connectivity issues, network instability, or storage service degradation that causes TiFlash to retry operations.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_system_profile_event_S3PutDMFileRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 181, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "IO Throughput", "group": "Server", "used_for": "Used for monitoring the I/O throughput rate (bytes per second) across TiFlash instances. Essential for identifying I/O bottlenecks, detecting abnormal disk activity, and diagnosing performance issues related to data read/write operations in TiFlash's storage layer.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 22, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Statement Num", "group": "Transaction", "used_for": "Used for monitoring the maximum number of SQL statements executed within a single transaction across TiDB instances. Helps identify transactions with excessive statement counts that may cause performance issues, long-running transactions, or potential application design problems requiring transaction splitting.", "component": "TIDB", "phenomenon": "io", "object": "query", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 180, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "S3 Request Duration", "group": "S3", "used_for": "Used for monitoring S3 request latency in TiFlash at different percentiles (p80, p90, p99, p100). Critical for diagnosing performance issues when TiFlash uses S3-compatible object storage, helping identify slow S3 operations that may impact query performance or data synchronization.", "component": "TIFLASH", "phenomenon": "latency", "object": "request", "percentile": [ 80, 90, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)", "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))", "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 147, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Region Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash region worker threads, which handle region data synchronization from TiKV. Essential for diagnosing performance bottlenecks in TiFlash's data replication process and identifying when region workers are CPU-saturated during heavy write workloads.", "component": "TIFLASH", "phenomenon": "cpu", "object": "region", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 17, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Execution Duration", "group": "Query Detail", "used_for": "Used for monitoring the 95th and 99th percentile execution duration of general SQL queries in TiDB. Essential for identifying query performance degradation and detecting slow query patterns that affect the majority of user requests.", "component": "TIDB", "phenomenon": "latency", "object": "query", "percentile": [ 95, 99 ], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", sql_type=\"general\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 185, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Operations", "group": "S3", "used_for": "Used for monitoring TiFlash's remote cache performance when using S3-compatible storage, tracking cache hit/miss rates for both DTFile and page-level operations. Essential for diagnosing I/O efficiency issues and optimizing cache configuration in disaggregated storage deployments.", "component": "TIFLASH", "phenomenon": "io", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit|dtfile_miss\"}[2m]))", "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit|page_miss\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 251, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Apply Worker", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of TiFlash's apply worker threads, which handle Raft log application and data replication from TiKV. Critical for diagnosing replication lag issues and identifying CPU bottlenecks in TiFlash's data synchronization process.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 23, "dashboard": "TiDB-Summary-Cluster-Clinic", "title": "Max Transaction Retry Num", "group": "Transaction", "used_for": "Used for monitoring the maximum number of retries for transactions in TiDB, representing the worst-case retry behavior across all transactions. Critical for identifying transaction conflicts, write contention issues, or optimistic locking problems that force transactions to retry multiple times before succeeding.", "component": "TIDB", "phenomenon": "io", "object": "transaction", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))" ], "dashboard_file": "tidb_summary.json" }, { "panel_id": 141, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Import Service", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST import service threads, which handle importing SST (Sorted String Table) files during data replication and recovery. Essential for diagnosing performance bottlenecks during bulk data ingestion, region migration, or when TiFlash replicas are being synchronized from TiKV.", "component": "TIFLASH", "phenomenon": "latency", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"sst_importer.*\"}[2m]))" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 154, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "SST Apply", "group": "Threads CPU", "used_for": "Used for monitoring CPU usage of TiFlash's SST apply threads, which handle low-priority Raft log applications to the storage engine. Essential for diagnosing write performance bottlenecks and identifying when apply threads are saturated during heavy write workloads or data replication.", "component": "TIFLASH", "phenomenon": "cpu", "object": "", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 155, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Raft Store", "group": "Threads CPU", "used_for": "Used for monitoring CPU utilization of Raft Store threads in TiFlash's proxy layer, which handle Raft consensus operations and data replication. Essential for diagnosing replication performance issues, identifying CPU bottlenecks in Raft processing, and detecting when Raft Store threads are overloaded or underutilized.", "component": "TIFLASH", "phenomenon": "cpu", "object": "store", "percentile": [], "variables": [ "k8s_cluster", "tidb_cluster" ], "promql": [ "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"}[2m]))", "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"})" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 186, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Remote Cache Flow", "group": "S3", "used_for": "Used for monitoring the rate of data flow through TiFlash's remote cache (S3-backed cache), broken down by operation type. Helps diagnose remote cache efficiency, identify cache hit/miss patterns, and troubleshoot performance issues related to S3 object storage access in disaggregated TiFlash deployments.", "component": "TIFLASH", "phenomenon": "", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "sum(rate(tiflash_storage_remote_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)" ], "dashboard_file": "tiflash-summary.json" }, { "panel_id": 99, "dashboard": "TiFlash-Summary-Cluster-Clinic", "title": "Rough Set Filter Rate", "group": "Rough Set Filter Rate Histogram", "used_for": "Used for monitoring the effectiveness of TiFlash's rough set filter in reducing data scan volume during query execution. Helps identify whether the rough set index is successfully filtering out unnecessary data blocks, with higher rates indicating better query performance optimization.", "component": "TIFLASH", "phenomenon": "throughput", "object": "", "percentile": [], "variables": [ "instance", "k8s_cluster", "tidb_cluster" ], "promql": [ "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (instance)", "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)", "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)" ], "dashboard_file": "tiflash-summary.json" } ]