[
  {
    "panel_id": 108,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Leader/Primary",
    "group": "",
    "used_for": "Used for monitoring PD leader/primary. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "service_member_role{service=~\"TSO Service Group .*\"}"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1214,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Running TiDB Pod Count",
    "group": "",
    "used_for": "Used for monitoring TiDB running tidb pod count.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(kube_pod_status_phase{namespace=~\".*tidb-pool\", phase=\"Running\"}) by (namespace)",
      "sum(kube_pod_status_phase{namespace=~\".*tidb-pool\", phase=\"Running\"})"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 207,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "CPU Usage",
    "group": "",
    "used_for": "Used for monitoring PD CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "PD",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(irate(process_cpu_seconds_total{job=~\".*tso.*\"}[1m])) by (instance, job)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 208,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Memory Usage",
    "group": "",
    "used_for": "Used for monitoring PD memory usage. Helps identify memory pressure and potential OOM risks.",
    "component": "PD",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(process_resident_memory_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_heap_sys_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_heap_inuse_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_heap_alloc_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_heap_idle_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_heap_released_bytes{job=~\".*tso.*\"}) by (instance, job)",
      "avg(go_memstats_next_gc_bytes{job=~\".*tso.*\") by (instance, job)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 209,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Uptime",
    "group": "",
    "used_for": "Used for monitoring PD uptime.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg((time() - process_start_time_seconds{job=~\".*tso.*\"})) by (instance, job)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 210,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Goroutine Count",
    "group": "",
    "used_for": "Used for monitoring PD goroutine count.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(go_goroutines{job=~\".*tso.*\"}) by (instance, job)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1207,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "TSO Request OPS",
    "group": "Request",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) tso request ops in the Request category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\"tso\"}[1m]))",
      "sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\"tso\"}[1m]))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1213,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "TSO  CMD Fail OPS",
    "group": "Request",
    "used_for": "Used for monitoring PD tso  cmd fail ops in the Request category. Helps detect failures and error patterns.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count{type=\"tso\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1209,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "TSO Wait Duration",
    "group": "Request",
    "used_for": "Used for monitoring PD tso wait duration in the Request category (P90/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      90,
      99,
      999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1211,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "TSO Async Wait Duration",
    "group": "Request",
    "used_for": "Used for monitoring PD tso async wait duration in the Request category (P90/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      90,
      99,
      999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1201,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "PD server TSO handle time",
    "group": "Handle",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) pd server tso handle time in the Handle category (P90/P99/P999/P99999/P9999999999). Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      99999,
      9999999999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.99999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.9999999999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1202,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "PD client TSO handle time",
    "group": "Handle",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) pd client tso handle time in the Handle category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_client_request_handle_requests_duration_seconds_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1203,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Handle requests count",
    "group": "Handle",
    "used_for": "Used for monitoring PD handle requests count in the Handle category.",
    "component": "PD",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tso_server_handle_tso_duration_seconds_count[1m])) by (type)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1204,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Tso Request batch size",
    "group": "Handle",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) tso request batch size in the Handle category (P90/P99). Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "request",
    "percentile": [
      90,
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket[1m])) by (le))",
      "sum(delta(pd_client_request_handle_tso_batch_size_sum{}[1m])) / sum(delta(pd_client_request_handle_tso_batch_size_count{}[1m]))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1206,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "TSO events rate",
    "group": "Handle",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) tso events rate in the Handle category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_tso_events[1m])) by (type)"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1217,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Keyspace Group State",
    "group": "Keyspace Group",
    "used_for": "Used for monitoring PD keyspace group state in the Keyspace Group category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tso_keyspace_group_state"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1219,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Keyspace Group Operation Count",
    "group": "Keyspace Group",
    "used_for": "Used for monitoring PD keyspace group operation count in the Keyspace Group category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tso_keyspace_group_operation_duration_seconds_count"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1218,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Keyspace Group Operation duration",
    "group": "Keyspace Group",
    "used_for": "Used for monitoring PD keyspace group operation duration in the Keyspace Group category (P90/P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.90, sum(rate(tso_keyspace_group_operation_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.99, sum(rate(tso_keyspace_group_operation_duration_seconds_bucket[1m])) by (type, le))"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 1220,
    "dashboard": "Starter-Essential-PD-TSO-for-Clinic",
    "title": "Keyspace Group Fallback Events Count",
    "group": "Keyspace Group",
    "used_for": "Used for monitoring PD keyspace group fallback events count in the Keyspace Group category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tso_keyspace_fallback_events"
    ],
    "dashboard_file": "pd-tso.json"
  },
  {
    "panel_id": 20,
    "dashboard": "Node Exporter Full",
    "title": "CPU Busy",
    "group": "",
    "used_for": "Used for monitoring node CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "(((count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))) - avg(sum by (mode)(rate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[$__rate_interval])))) * 100) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 155,
    "dashboard": "Node Exporter Full",
    "title": "Sys Load (5m avg)",
    "group": "",
    "used_for": "Used for monitoring node sys load (5m avg).",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "avg(node_load5{instance=\"$node\",job=\"$job\"}) /  count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 19,
    "dashboard": "Node Exporter Full",
    "title": "Sys Load (15m avg)",
    "group": "",
    "used_for": "Used for monitoring node sys load (15m avg).",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "avg(node_load15{instance=\"$node\",job=\"$job\"}) /  count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 16,
    "dashboard": "Node Exporter Full",
    "title": "RAM Used",
    "group": "",
    "used_for": "Used for monitoring node ram used.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100",
      "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 21,
    "dashboard": "Node Exporter Full",
    "title": "SWAP Used",
    "group": "",
    "used_for": "Used for monitoring node swap used.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 154,
    "dashboard": "Node Exporter Full",
    "title": "Root FS Used",
    "group": "",
    "used_for": "Used for monitoring node root fs used.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 14,
    "dashboard": "Node Exporter Full",
    "title": "CPU Cores",
    "group": "",
    "used_for": "Used for monitoring node CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 15,
    "dashboard": "Node Exporter Full",
    "title": "Uptime",
    "group": "",
    "used_for": "Used for monitoring node uptime.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 23,
    "dashboard": "Node Exporter Full",
    "title": "RootFS Total",
    "group": "",
    "used_for": "Used for monitoring node rootfs total.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 75,
    "dashboard": "Node Exporter Full",
    "title": "RAM Total",
    "group": "",
    "used_for": "Used for monitoring node ram total.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 18,
    "dashboard": "Node Exporter Full",
    "title": "SWAP Total",
    "group": "",
    "used_for": "Used for monitoring node swap total.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 77,
    "dashboard": "Node Exporter Full",
    "title": "CPU Basic",
    "group": "",
    "used_for": "Used for monitoring node CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "sum by (instance)(rate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (instance)(rate(node_cpu_seconds_total{mode='user',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (instance)(rate(node_cpu_seconds_total{mode='iowait',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (instance)(rate(node_cpu_seconds_total{mode=~\".*irq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum (rate(node_cpu_seconds_total{mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 78,
    "dashboard": "Node Exporter Full",
    "title": "Memory Basic",
    "group": "",
    "used_for": "Used for monitoring node memory usage. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})",
      "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
      "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 74,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Basic",
    "group": "",
    "used_for": "Used for monitoring node network traffic basic. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
      "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 152,
    "dashboard": "Node Exporter Full",
    "title": "Disk Space Used Basic",
    "group": "",
    "used_for": "Used for monitoring node disk space used basic. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 3,
    "dashboard": "Node Exporter Full",
    "title": "CPU",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node CPU utilization in the CPU / Memory / Net / Disk category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "sum by (mode)(rate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='user',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='nice',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='iowait',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='irq',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='softirq',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='steal',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='guest',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100",
      "sum by (mode)(rate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[$__rate_interval])) * 100"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 24,
    "dashboard": "Node Exporter Full",
    "title": "Memory Stack",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node memory usage in the CPU / Memory / Net / Disk category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}",
      "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})",
      "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 84,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node network traffic in the CPU / Memory / Net / Disk category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8",
      "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 156,
    "dashboard": "Node Exporter Full",
    "title": "Disk Space Used",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node disk space used in the CPU / Memory / Net / Disk category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 229,
    "dashboard": "Node Exporter Full",
    "title": "Disk IOps",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node disk iops in the CPU / Memory / Net / Disk category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 42,
    "dashboard": "Node Exporter Full",
    "title": "I/O Usage Read / Write",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node i/o usage read / write in the CPU / Memory / Net / Disk category.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 127,
    "dashboard": "Node Exporter Full",
    "title": "I/O Utilization",
    "group": "CPU / Memory / Net / Disk",
    "used_for": "Used for monitoring node i/o utilization in the CPU / Memory / Net / Disk category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"} [$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 136,
    "dashboard": "Node Exporter Full",
    "title": "Memory Active / Inactive",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 135,
    "dashboard": "Node Exporter Full",
    "title": "Memory Commited",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 191,
    "dashboard": "Node Exporter Full",
    "title": "Memory Active / Inactive Detail",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 130,
    "dashboard": "Node Exporter Full",
    "title": "Memory Writeback and Dirty",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 138,
    "dashboard": "Node Exporter Full",
    "title": "Memory Shared and Mapped",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 131,
    "dashboard": "Node Exporter Full",
    "title": "Memory Slab",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 70,
    "dashboard": "Node Exporter Full",
    "title": "Memory Vmalloc",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 159,
    "dashboard": "Node Exporter Full",
    "title": "Memory Bounce",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 129,
    "dashboard": "Node Exporter Full",
    "title": "Memory Anonymous",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 160,
    "dashboard": "Node Exporter Full",
    "title": "Memory Kernel / CPU",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node CPU utilization in the Memory Meminfo category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 140,
    "dashboard": "Node Exporter Full",
    "title": "Memory HugePages Counter",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}",
      "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}",
      "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 71,
    "dashboard": "Node Exporter Full",
    "title": "Memory HugePages Size",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}",
      "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 128,
    "dashboard": "Node Exporter Full",
    "title": "Memory DirectMap",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 137,
    "dashboard": "Node Exporter Full",
    "title": "Memory Unevictable and MLocked",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}",
      "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 132,
    "dashboard": "Node Exporter Full",
    "title": "Memory NFS",
    "group": "Memory Meminfo",
    "used_for": "Used for monitoring node memory usage in the Memory Meminfo category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 176,
    "dashboard": "Node Exporter Full",
    "title": "Memory Pages In / Out",
    "group": "Memory Vmstat",
    "used_for": "Used for monitoring node memory usage in the Memory Vmstat category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 22,
    "dashboard": "Node Exporter Full",
    "title": "Memory Pages Swap In / Out",
    "group": "Memory Vmstat",
    "used_for": "Used for monitoring node memory usage in the Memory Vmstat category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 175,
    "dashboard": "Node Exporter Full",
    "title": "Memory Page Faults",
    "group": "Memory Vmstat",
    "used_for": "Used for monitoring node memory usage in the Memory Vmstat category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])  - rate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 307,
    "dashboard": "Node Exporter Full",
    "title": "OOM Killer",
    "group": "Memory Vmstat",
    "used_for": "Used for monitoring node oom killer in the Memory Vmstat category.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 260,
    "dashboard": "Node Exporter Full",
    "title": "Time Syncronized Drift",
    "group": "System Timesync",
    "used_for": "Used for monitoring node time syncronized drift in the System Timesync category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}",
      "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}",
      "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 291,
    "dashboard": "Node Exporter Full",
    "title": "Time PLL Adjust",
    "group": "System Timesync",
    "used_for": "Used for monitoring node time pll adjust in the System Timesync category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 168,
    "dashboard": "Node Exporter Full",
    "title": "Time Syncronized Status",
    "group": "System Timesync",
    "used_for": "Used for monitoring node time syncronized status in the System Timesync category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_timex_sync_status{instance=\"$node\",job=\"$job\"}",
      "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 294,
    "dashboard": "Node Exporter Full",
    "title": "Time Misc",
    "group": "System Timesync",
    "used_for": "Used for monitoring node time misc in the System Timesync category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}",
      "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 62,
    "dashboard": "Node Exporter Full",
    "title": "Processes Status",
    "group": "System Processes",
    "used_for": "Used for monitoring node processes status in the System Processes category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_procs_blocked{instance=\"$node\",job=\"$job\"}",
      "node_procs_running{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 315,
    "dashboard": "Node Exporter Full",
    "title": "Processes State",
    "group": "System Processes",
    "used_for": "Used for monitoring node processes state in the System Processes category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_processes_state{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 148,
    "dashboard": "Node Exporter Full",
    "title": "Processes  Forks",
    "group": "System Processes",
    "used_for": "Used for monitoring node processes  forks in the System Processes category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 149,
    "dashboard": "Node Exporter Full",
    "title": "Processes Memory",
    "group": "System Processes",
    "used_for": "Used for monitoring node memory usage in the System Processes category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}",
      "rate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 313,
    "dashboard": "Node Exporter Full",
    "title": "PIDs Number and Limit",
    "group": "System Processes",
    "used_for": "Used for monitoring node pids number and limit in the System Processes category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_processes_pids{instance=\"$node\",job=\"$job\"}",
      "node_processes_max_processes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 305,
    "dashboard": "Node Exporter Full",
    "title": "Process schedule stats Running / Waiting",
    "group": "System Processes",
    "used_for": "Used for monitoring node process schedule stats running / waiting in the System Processes category. Helps track scheduling operations and potential bottlenecks.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 314,
    "dashboard": "Node Exporter Full",
    "title": "Threads Number and Limit",
    "group": "System Processes",
    "used_for": "Used for monitoring node threads number and limit in the System Processes category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_processes_threads{instance=\"$node\",job=\"$job\"}",
      "node_processes_max_threads{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 8,
    "dashboard": "Node Exporter Full",
    "title": "Context Switches / Interrupts",
    "group": "System Misc",
    "used_for": "Used for monitoring node context switches / interrupts in the System Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 7,
    "dashboard": "Node Exporter Full",
    "title": "System Load",
    "group": "System Misc",
    "used_for": "Used for monitoring node system load in the System Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_load1{instance=\"$node\",job=\"$job\"}",
      "node_load5{instance=\"$node\",job=\"$job\"}",
      "node_load15{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 259,
    "dashboard": "Node Exporter Full",
    "title": "Interrupts Detail",
    "group": "System Misc",
    "used_for": "Used for monitoring node interrupts detail in the System Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 306,
    "dashboard": "Node Exporter Full",
    "title": "Schedule timeslices executed by each cpu",
    "group": "System Misc",
    "used_for": "Used for monitoring node CPU utilization in the System Misc category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 151,
    "dashboard": "Node Exporter Full",
    "title": "Entropy",
    "group": "System Misc",
    "used_for": "Used for monitoring node entropy in the System Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 308,
    "dashboard": "Node Exporter Full",
    "title": "CPU time spent in user and system contexts",
    "group": "System Misc",
    "used_for": "Used for monitoring node CPU utilization in the System Misc category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 64,
    "dashboard": "Node Exporter Full",
    "title": "File Descriptors",
    "group": "System Misc",
    "used_for": "Used for monitoring node file descriptors in the System Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "process_max_fds{instance=\"$node\",job=\"$job\"}",
      "process_open_fds{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 158,
    "dashboard": "Node Exporter Full",
    "title": "Hardware temperature monitor",
    "group": "Hardware Misc",
    "used_for": "Used for monitoring node hardware temperature monitor in the Hardware Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}",
      "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}",
      "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}",
      "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}",
      "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 300,
    "dashboard": "Node Exporter Full",
    "title": "Throttle cooling device",
    "group": "Hardware Misc",
    "used_for": "Used for monitoring node throttle cooling device in the Hardware Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}",
      "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 302,
    "dashboard": "Node Exporter Full",
    "title": "Power supply",
    "group": "Hardware Misc",
    "used_for": "Used for monitoring node power supply in the Hardware Misc category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_power_supply_online{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 297,
    "dashboard": "Node Exporter Full",
    "title": "Systemd Sockets",
    "group": "Systemd",
    "used_for": "Used for monitoring node systemd sockets in the Systemd category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 298,
    "dashboard": "Node Exporter Full",
    "title": "Systemd Units State",
    "group": "Systemd",
    "used_for": "Used for monitoring node systemd units state in the Systemd category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}",
      "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}",
      "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}",
      "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}",
      "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 9,
    "dashboard": "Node Exporter Full",
    "title": "Disk IOps Completed",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node disk iops completed in the Storage Disk category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 33,
    "dashboard": "Node Exporter Full",
    "title": "Disk R/W Data",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node disk r/w data in the Storage Disk category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 37,
    "dashboard": "Node Exporter Full",
    "title": "Disk Average Wait Time",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node disk average wait time in the Storage Disk category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 35,
    "dashboard": "Node Exporter Full",
    "title": "Average Queue Size",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node average queue size in the Storage Disk category.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 133,
    "dashboard": "Node Exporter Full",
    "title": "Disk R/W Merged",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node disk r/w merged in the Storage Disk category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 36,
    "dashboard": "Node Exporter Full",
    "title": "Time Spent Doing I/Os",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node time spent doing i/os in the Storage Disk category.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 34,
    "dashboard": "Node Exporter Full",
    "title": "Instantaneous Queue Size",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node instantaneous queue size in the Storage Disk category.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_disk_io_now{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 301,
    "dashboard": "Node Exporter Full",
    "title": "Disk IOps Discards completed / merged",
    "group": "Storage Disk",
    "used_for": "Used for monitoring node disk iops discards completed / merged in the Storage Disk category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 43,
    "dashboard": "Node Exporter Full",
    "title": "Filesystem space available",
    "group": "Storage Filesystem",
    "used_for": "Used for monitoring node filesystem space available in the Storage Filesystem category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
      "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
      "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 41,
    "dashboard": "Node Exporter Full",
    "title": "File Nodes Free",
    "group": "Storage Filesystem",
    "used_for": "Used for monitoring node file nodes free in the Storage Filesystem category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 28,
    "dashboard": "Node Exporter Full",
    "title": "File Descriptor",
    "group": "Storage Filesystem",
    "used_for": "Used for monitoring node file descriptor in the Storage Filesystem category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filefd_maximum{instance=\"$node\",job=\"$job\"}",
      "node_filefd_allocated{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 219,
    "dashboard": "Node Exporter Full",
    "title": "File Nodes Size",
    "group": "Storage Filesystem",
    "used_for": "Used for monitoring node file nodes size in the Storage Filesystem category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 44,
    "dashboard": "Node Exporter Full",
    "title": "Filesystem in ReadOnly / Error",
    "group": "Storage Filesystem",
    "used_for": "Used for monitoring node filesystem in readonly / error in the Storage Filesystem category. Helps detect failures and error patterns.",
    "component": "",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}",
      "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 60,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic by Packets",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic by packets in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 142,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Errors",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic errors in the Network Traffic category. Helps detect failures and error patterns.",
    "component": "",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 143,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Drop",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic drop in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 141,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Compressed",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic compressed in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 146,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Multicast",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic multicast in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 144,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Fifo",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic fifo in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 145,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Frame",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic frame in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 231,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Carrier",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic carrier in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 232,
    "dashboard": "Node Exporter Full",
    "title": "Network Traffic Colls",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network traffic colls in the Network Traffic category. Helps identify network throughput and bandwidth issues.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 61,
    "dashboard": "Node Exporter Full",
    "title": "NF Contrack",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node nf contrack in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}",
      "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 230,
    "dashboard": "Node Exporter Full",
    "title": "ARP Entries",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node arp entries in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_arp_entries{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 288,
    "dashboard": "Node Exporter Full",
    "title": "MTU",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node mtu in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 280,
    "dashboard": "Node Exporter Full",
    "title": "Speed",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node speed in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_network_speed_bytes{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 289,
    "dashboard": "Node Exporter Full",
    "title": "Queue Length",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node queue length in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 290,
    "dashboard": "Node Exporter Full",
    "title": "Softnet Packets",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node softnet packets in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 310,
    "dashboard": "Node Exporter Full",
    "title": "Softnet Out of Quota",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node softnet out of quota in the Network Traffic category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 309,
    "dashboard": "Node Exporter Full",
    "title": "Network Operational Status",
    "group": "Network Traffic",
    "used_for": "Used for monitoring node network operational status in the Network Traffic category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}",
      "node_network_carrier{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 63,
    "dashboard": "Node Exporter Full",
    "title": "Sockstat TCP",
    "group": "Network Sockstat",
    "used_for": "Used for monitoring node sockstat tcp in the Network Sockstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 124,
    "dashboard": "Node Exporter Full",
    "title": "Sockstat UDP",
    "group": "Network Sockstat",
    "used_for": "Used for monitoring node sockstat udp in the Network Sockstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 125,
    "dashboard": "Node Exporter Full",
    "title": "Sockstat FRAG / RAW",
    "group": "Network Sockstat",
    "used_for": "Used for monitoring node sockstat frag / raw in the Network Sockstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 220,
    "dashboard": "Node Exporter Full",
    "title": "Sockstat Memory Size",
    "group": "Network Sockstat",
    "used_for": "Used for monitoring node memory usage in the Network Sockstat category. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}",
      "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 126,
    "dashboard": "Node Exporter Full",
    "title": "Sockstat Used",
    "group": "Network Sockstat",
    "used_for": "Used for monitoring node sockstat used in the Network Sockstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 221,
    "dashboard": "Node Exporter Full",
    "title": "Netstat IP In / Out Octets",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node netstat ip in / out octets in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 81,
    "dashboard": "Node Exporter Full",
    "title": "Netstat IP Forwarding",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node netstat ip forwarding in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 115,
    "dashboard": "Node Exporter Full",
    "title": "ICMP In / Out",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node icmp in / out in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 50,
    "dashboard": "Node Exporter Full",
    "title": "ICMP Errors",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node icmp errors in the Network Netstat category. Helps detect failures and error patterns.",
    "component": "",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 55,
    "dashboard": "Node Exporter Full",
    "title": "UDP In / Out",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node udp in / out in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 109,
    "dashboard": "Node Exporter Full",
    "title": "UDP Errors",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node udp errors in the Network Netstat category. Helps detect failures and error patterns.",
    "component": "",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 299,
    "dashboard": "Node Exporter Full",
    "title": "TCP In / Out",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node tcp in / out in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 104,
    "dashboard": "Node Exporter Full",
    "title": "TCP Errors",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node tcp errors in the Network Netstat category. Helps detect failures and error patterns.",
    "component": "",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 85,
    "dashboard": "Node Exporter Full",
    "title": "TCP Connections",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node tcp connections in the Network Netstat category. Helps track connection patterns and potential connection exhaustion.",
    "component": "",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}",
      "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 91,
    "dashboard": "Node Exporter Full",
    "title": "TCP SynCookie",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node tcp syncookie in the Network Netstat category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 82,
    "dashboard": "Node Exporter Full",
    "title": "TCP Direct Transition",
    "group": "Network Netstat",
    "used_for": "Used for monitoring node tcp direct transition in the Network Netstat category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "job",
      "node"
    ],
    "promql": [
      "rate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])",
      "rate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 40,
    "dashboard": "Node Exporter Full",
    "title": "Node Exporter Scrape Time",
    "group": "Node Exporter",
    "used_for": "Used for monitoring node node exporter scrape time in the Node Exporter category.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 157,
    "dashboard": "Node Exporter Full",
    "title": "Node Exporter Scrape",
    "group": "Node Exporter",
    "used_for": "Used for monitoring node node exporter scrape in the Node Exporter category.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "job",
      "node"
    ],
    "promql": [
      "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}",
      "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 317,
    "dashboard": "Node Exporter Full",
    "title": "TiKV node vs pod",
    "group": "Others",
    "used_for": "Used for monitoring TiKV tikv node vs pod in the Others category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "count(kube_pod_info{namespace=\"tidb-serverless\",pod=~\"tikv.*\",pod!~\"tikv-worker.*|tikv-api-.*\"} * on(node) group_left(internal_ip) kube_node_info) by (pod, node)"
    ],
    "dashboard_file": "node-info.json"
  },
  {
    "panel_id": 100,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Storage capacity",
    "group": "",
    "used_for": "Used for monitoring PD storage capacity. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"storage_capacity\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 101,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current storage size",
    "group": "",
    "used_for": "Used for monitoring PD current storage size. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"storage_size\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 102,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current storage used",
    "group": "",
    "used_for": "Used for monitoring PD current storage used. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"storage_size\"}) / sum(pd_cluster_status{type=\"storage_capacity\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 103,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Normal stores",
    "group": "",
    "used_for": "Used for monitoring PD normal stores. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"store_up_count\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 104,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Number of Regions",
    "group": "",
    "used_for": "Used for monitoring PD number of regions. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"leader_count\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 105,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current peer count",
    "group": "",
    "used_for": "Used for monitoring PD current peer count.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"region_count\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 106,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Abnormal stores",
    "group": "",
    "used_for": "Used for monitoring PD abnormal stores. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_status{type=\"store_disconnected_count\"})",
      "sum(pd_cluster_status{type=\"store_unhealth_count\"})",
      "sum(pd_cluster_status{type=\"store_low_space_count\"})",
      "sum(pd_cluster_status{type=\"store_down_count\"})",
      "sum(pd_cluster_status{type=\"store_offline_count\"})",
      "sum(pd_cluster_status{type=\"store_tombstone_count\"})",
      "sum(pd_cluster_status{type=\"store_slow_count\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 107,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region health",
    "group": "",
    "used_for": "Used for monitoring PD region health. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(pd_regions_status) by (type)",
      "sum(pd_regions_status) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 108,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Leader/Primary",
    "group": "",
    "used_for": "Used for monitoring PD leader/primary. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "service_member_role"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 109,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Offline store progress",
    "group": "",
    "used_for": "Used for monitoring PD offline store progress. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(pd_cluster_progress{action=\"removing\"} > 0)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 110,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Online store progress",
    "group": "",
    "used_for": "Used for monitoring PD online store progress. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(pd_cluster_progress{action=\"preparing\"} > 0)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 111,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Left time",
    "group": "",
    "used_for": "Used for monitoring PD left time.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_eta"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 112,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current scaling speed",
    "group": "",
    "used_for": "Used for monitoring PD current scaling speed.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_speed"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 113,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Placement Rules Status",
    "group": "",
    "used_for": "Used for monitoring PD placement rules status.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_rule_manager_status) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 201,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "PD scheduler config",
    "group": "Cluster",
    "used_for": "Used for monitoring PD pd scheduler config in the Cluster category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "pd_config_status{instance=~\"$instance\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 202,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store limit",
    "group": "Cluster",
    "used_for": "Used for monitoring PD store limit in the Cluster category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_store_limit"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 203,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Cluster ID",
    "group": "Cluster",
    "used_for": "Used for monitoring PD cluster id in the Cluster category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_metadata{instance=~\".*pd.*\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 204,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region label isolation level",
    "group": "Cluster",
    "used_for": "Used for monitoring PD region label isolation level in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "pd_regions_label_level{instance=~\"$instance\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 205,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Label distribution",
    "group": "Cluster",
    "used_for": "Used for monitoring PD label distribution in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_placement_status"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 206,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current ID allocation",
    "group": "Cluster",
    "used_for": "Used for monitoring PD current id allocation in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_cluster_id{type=\"idalloc\"} != 0"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 207,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "CPU Usage",
    "group": "Cluster",
    "used_for": "Used for monitoring PD CPU utilization in the Cluster category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "PD",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(irate(process_cpu_seconds_total{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}[30s])) by (instance, job)",
      "pd_service_maxprocs{job=~\".*pd.*\"}",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"}[1m])) by (instance)",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-standby-pd.*\"}[1m])) by (instance)",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-witness-pd.*\"}[1m])) by (instance)",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-tso-server.*\"}[1m])) by (instance)",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-resource-manager-server.*\"}[1m])) by (instance)",
      "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-sche-server.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 208,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Memory Usage",
    "group": "Cluster",
    "used_for": "Used for monitoring PD memory usage in the Cluster category. Helps identify memory pressure and potential OOM risks.",
    "component": "PD",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(process_resident_memory_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_heap_sys_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_heap_inuse_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_heap_alloc_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_heap_idle_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_heap_released_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)",
      "avg(go_memstats_next_gc_bytes{job=~\".*pd.*|.*tso.*|.*sche.*|.*resource-manager.*\"}) by (instance, job)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 210,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Goroutine Count",
    "group": "Cluster",
    "used_for": "Used for monitoring PD goroutine count in the Cluster category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(go_goroutines{job=~\".*pd.*|.*tso.*|.*sche.*\"}) by (instance, job)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 209,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Uptime",
    "group": "Cluster",
    "used_for": "Used for monitoring PD uptime in the Cluster category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg((time() - process_start_time_seconds{job=~\".*pd.*|.*tso.*|.*sche.*\"})) by (instance, job)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1620,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Disk Read/Write",
    "group": "Cluster",
    "used_for": "Used for monitoring PD disk read/write in the Cluster category. Helps identify disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(irate(container_fs_writes_bytes_total{pod=~\".*pd.*\", namespace=\"tidb-serverless\"}[1m])) by (pod)",
      "sum(irate(container_fs_reads_bytes_total{pod=~\".*pd.*\", namespace=\"tidb-serverless\"}[1m])) by (pod)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1622,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Disk IOPS",
    "group": "Cluster",
    "used_for": "Used for monitoring PD disk iops in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(\n  irate(container_fs_reads_total{pod=~\".*pd.*\", namespace=\"tidb-serverless\"}[1m]) + \n  irate(container_fs_writes_total{pod=~\".*pd.*\", namespace=\"tidb-serverless\"}[1m])\n) by (pod)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 211,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current TSO Physcial",
    "group": "Cluster",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) current tso physcial in the Cluster category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 212,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Max TSO gap",
    "group": "Cluster",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) max tso gap in the Cluster category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(pd_cluster_tso_gap_millionseconds) by (dc)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 213,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Scheduling Allowance Status",
    "group": "Cluster",
    "used_for": "Used for monitoring PD scheduling allowance status in the Cluster category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_scheduling_allowance_status"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 214,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Current TSO Datetime",
    "group": "Cluster",
    "used_for": "Used for monitoring PD TSO (Timestamp Oracle) current tso datetime in the Cluster category. Helps track timestamp allocation performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 301,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operator create",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operator create in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count{event=\"create\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 302,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operator check",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operator check in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count{event=\"check\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 303,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operator finish",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operator finish in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count{event=\"finish\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 304,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operator timeout",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operator timeout in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count{event=\"timeout\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 305,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operator replaced or canceled",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operator replaced or canceled in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count{event=\"cancel\"}[1m])) by (type)",
      "sum(delta(pd_schedule_operators_count{event=\"replace\"}[1m])) by (type)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 306,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedule operators count by state",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedule operators count by state in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operators_count[1m])) by (event)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 307,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Operator finish duration",
    "group": "Operator",
    "used_for": "Used for monitoring PD operator finish duration in the Operator category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))",
      "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 308,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Operator step finish duration",
    "group": "Operator",
    "used_for": "Used for monitoring PD operator step finish duration in the Operator category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))",
      "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 309,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Schedulers reach limit",
    "group": "Operator",
    "used_for": "Used for monitoring PD schedulers reach limit in the Operator category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_operator_limit[1m])) by (type,name)",
      "sum(delta(pd_schedule_operator_exceeded_store_limit[1m])) by (desc)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 310,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% operator region size",
    "group": "Operator",
    "used_for": "Used for monitoring PD 99% operator region size in the Operator category (P99). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket[5m])) by (type,le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 401,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store capacity",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store capacity in the Statistics - balance category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_capacity\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 403,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store used",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store used in the Statistics - balance category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_used\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 405,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Size amplification",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD size amplification in the Statistics - balance category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(pd_scheduler_store_status{store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{type=\"store_used\"}) by (address, store) * 2^20"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 407,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store leader score",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store leader score in the Statistics - balance category. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"leader_score\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 409,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store leader size",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store leader size in the Statistics - balance category. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"leader_size\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 412,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store leader count",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store leader count in the Statistics - balance category. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"leader_count\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 402,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store available",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store available in the Statistics - balance category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "{store=~\"$store\", type=\"store_available\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 404,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store available ratio",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store available ratio in the Statistics - balance category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(pd_scheduler_store_status{store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{type=\"store_capacity\"}) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 406,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store Region score",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store region score in the Statistics - balance category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"region_score\"}",
      "pd_scheduler_op_influence{scheduler=\"balance-region-scheduler\"}",
      "pd_scheduler_tolerant_resource{scheduler=\"balance-region-scheduler\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 408,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store Region size",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store region size in the Statistics - balance category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"region_size\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 410,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store Region count",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store region count in the Statistics - balance category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"region_count\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 411,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store witness count",
    "group": "Statistics - balance",
    "used_for": "Used for monitoring PD store witness count in the Statistics - balance category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"witness_count\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 501,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot Region's leader distribution",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD hot region's leader distribution in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotspot_status{store=~\"$store\", type=\"hot_write_region_as_leader\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 502,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot Region's peer distribution",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD hot region's peer distribution in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotspot_status{store=~\"$store\", type=\"hot_write_region_as_peer\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 503,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store write rate bytes",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD store write rate bytes in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_bytes\"}",
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_regions_write_rate_bytes\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 504,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Total written keys on hot leader Regions",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD total written keys on hot leader regions in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotspot_status{store=~\"$store\", type=\"total_write_keys_as_leader\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 506,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store write rate keys",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD store write rate keys in the Statistics - hot write category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_keys\"}",
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_regions_write_rate_keys\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 505,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Total written bytes on hot leader Regions",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD total written bytes on hot leader regions in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotspot_status{store=~\"$store\", type=\"total_write_bytes_as_leader\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 508,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot cache write entry numbers",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD hot cache write entry numbers in the Statistics - hot write category. Helps evaluate cache effectiveness and hit rates.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotcache_status{name=\"total_length\", store=~\"store-$store\", type=\"write\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 507,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store write query",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD store write query in the Statistics - hot write category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "io",
    "object": "query",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_query_rate\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 510,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Direction of hotspot transfer leader",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD direction of hotspot transfer leader in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "- sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[1m]))by (store)",
      "sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[1m]))by (store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 509,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Selector write events",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD selector write events in the Statistics - hot write category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "rate(pd_scheduler_hot_region{store=~\"$store\", type=~\".*write.*\"}[1m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 512,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Remaining items in Queue",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD remaining items in queue in the Statistics - hot write category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_hotcache_flow_queue_status{type=\"write\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 511,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Direction of hotspot move peer",
    "group": "Statistics - hot write",
    "used_for": "Used for monitoring PD direction of hotspot move peer in the Statistics - hot write category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "- sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[1m]))by (store)",
      "sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[1m]))by (store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 601,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot Region's peer distribution",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD hot region's peer distribution in the Statistics - hot read category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotspot_status{store=~\"$store\", type=\"hot_read_region_as_peer\"}",
      "pd_hotspot_status{store=~\"$store\", type=\"hot_read_region_as_leader\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 602,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store read rate bytes",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD store read rate bytes in the Statistics - hot read category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_bytes\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 603,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store read rate keys",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD store read rate keys in the Statistics - hot read category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "store",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_rate_keys\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 604,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store read query",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD store read query in the Statistics - hot read category. Helps track storage capacity and usage patterns.",
    "component": "PD",
    "phenomenon": "io",
    "object": "query",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_scheduler_store_status{store=~\"$store\", type=\"store_read_query_rate\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}",
      "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 605,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Direction of hotspot transfer leader",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD direction of hotspot transfer leader in the Statistics - hot read category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "- sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[1m]))by (store)",
      "sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[1m]))by (store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 606,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Direction of hotspot move peer",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD direction of hotspot move peer in the Statistics - hot read category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "k8s_cluster",
      "store",
      "tidb_cluster"
    ],
    "promql": [
      "- sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[1m]))by (store)",
      "sum(delta(pd_scheduler_hot_region_direction{store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[1m]))by (store)",
      "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[1m]))by (store)",
      "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[1m]))by (store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 607,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot cache read entry numbers",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD hot cache read entry numbers in the Statistics - hot read category. Helps evaluate cache effectiveness and hit rates.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "pd_hotcache_status{name=\"total_length\", store=~\"store-$store\", type=\"read\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 608,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Remaining items in Queue",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD remaining items in queue in the Statistics - hot read category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_hotcache_flow_queue_status{type=\"read\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 609,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Selector read events",
    "group": "Statistics - hot read",
    "used_for": "Used for monitoring PD selector read events in the Statistics - hot read category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "rate(pd_scheduler_hot_region{store=~\"$store\", type=~\".*read.*\"}[1m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 701,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Scheduler is running",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD scheduler is running in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_scheduler_status{type=\"allow\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 702,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance leader movement",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance leader movement in the Scheduler category. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "-sum(delta(pd_scheduler_balance_direction{type=\"balance-leader-scheduler\"}[1m])) by (source)",
      "sum(delta(pd_scheduler_balance_direction{type=\"balance-leader-scheduler\"}[1m])) by (target)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 703,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance Region movement",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance region movement in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "-sum(delta(pd_scheduler_balance_direction{type=\"balance-region-scheduler\"}[1m])) by (source)",
      "sum(delta(pd_scheduler_balance_direction{type=\"balance-region-scheduler\"}[1m])) by (target)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 704,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance leader scheduler",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance leader scheduler in the Scheduler category. Helps track leader distribution and election status.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_scheduler_event_count{type=\"balance-leader-scheduler\"}[5m])) by (name)",
      "sum(rate(pd_scheduler_event_count{type=\"balance-leader-scheduler\"}[5m]))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 705,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance Region scheduler",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance region scheduler in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_scheduler_event_count{type=\"balance-region-scheduler\"}[5m])) by (name)",
      "sum(rate(pd_scheduler_event_count{type=\"balance-region-scheduler\"}[5m]))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 706,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance Hot Region scheduler",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance hot region scheduler in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_scheduler_event_count{type=\"balance-hot-region-scheduler\"}[5m])) by (name)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 707,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Split Bucket scheduler",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD split bucket scheduler in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_scheduler_event_count{type=\"split-bucket-scheduler\"}[5m])) by (name)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 708,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Balance Direction",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD balance direction in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_balance_direction[1m])) by (type, source, target)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 709,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Patrol Region time",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD patrol region time in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_checker_patrol_regions_time != 0"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 710,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Rule checker",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD rule checker in the Scheduler category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_checker_event_count{type=\"rule_checker\"}[1m])) by (name)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 711,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Replica checker",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD replica checker in the Scheduler category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_checker_event_count{type=\"replica_checker\"}[1m])) by (name)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 712,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region merge checker",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD region merge checker in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_checker_event_count{type=\"merge_checker\"}[1m])) by (name)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 713,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Filter source",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD filter source in the Scheduler category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_schedule_filter{store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[1m])) by (source, type, scope)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 714,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Filter target",
    "group": "Scheduler",
    "used_for": "Used for monitoring PD filter target in the Scheduler category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "k8s_cluster",
      "store",
      "tidb_cluster"
    ],
    "promql": [
      "sum(delta(pd_schedule_filter{store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[1m])) by (target, type, scope)",
      "sum(delta(pd_schedule_filter{action=\"filter-target\",type=\"distinct-filter\"}[1m])) by (source, target, type, scope)",
      "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[1m])) by (source, target, type, scope)",
      "sum(delta(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[1m])) by (source, target, type, scope)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 801,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "scatter operator event",
    "group": "Scatter and Splitter",
    "used_for": "Used for monitoring PD scatter operator event in the Scatter and Splitter category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_scatter_operators_count{type=\"skip\"}[1m])) by (event)",
      "delta(pd_schedule_scatter_operators_count{type=\"fail\"}[1m])",
      "delta(pd_schedule_scatter_operators_count{type=\"success\"}[1m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 802,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "scatter store selection",
    "group": "Scatter and Splitter",
    "used_for": "Used for monitoring PD scatter store selection in the Scatter and Splitter category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_schedule_scatter_distribution{engine=\"tikv\", is_leader=\"false\"}[1m])) by (store)",
      "sum(delta(pd_schedule_scatter_distribution{engine=\"tikv\", is_leader=\"true\"}[1m])) by (store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 901,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "gRPC Completed commands rate",
    "group": "gRPC",
    "used_for": "Used for monitoring PD grpc completed commands rate in the gRPC category. Helps track gRPC request performance between components.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(grpc_server_handling_seconds_count{}[1m])) by (grpc_method)",
      "sum(rate(grpc_server_handling_seconds_count{grpc_method=\"Txn\"}[1m]))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 902,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "gRPC 99% Completed commands duration",
    "group": "gRPC",
    "used_for": "Used for monitoring PD grpc 99% completed commands duration in the gRPC category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{}[5m])) by (grpc_method, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 903,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "gRPC commands concurrency number",
    "group": "gRPC",
    "used_for": "Used for monitoring PD grpc commands concurrency number in the gRPC category. Helps track gRPC request performance between components.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_server_api_concurrency{kind=\"grpc\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1001,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "HTTP Completed commands rate",
    "group": "HTTP",
    "used_for": "Used for monitoring PD http completed commands rate in the HTTP category.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_service_audit_handling_seconds_count{}[1m])) by (service,ip)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1002,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "HTTP 99% Completed commands duration",
    "group": "HTTP",
    "used_for": "Used for monitoring PD http 99% completed commands duration in the HTTP category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{}[5m])) by (service, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1003,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "HTTP commands concurrency number",
    "group": "HTTP",
    "used_for": "Used for monitoring PD http commands concurrency number in the HTTP category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_server_api_concurrency{kind=\"http\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1101,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC DB total size",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc db total size in the etcd category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_mvcc_db_total_size_in_bytes{component=\"pd\"}",
      "etcd_mvcc_db_total_size_in_use_in_bytes{component=\"pd\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1102,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Handle transactions rate",
    "group": "etcd",
    "used_for": "Used for monitoring PD handle transactions rate in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_txn_handle_txns_duration_seconds_count[5m])) by (instance, result)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1103,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% Handle transactions duration",
    "group": "etcd",
    "used_for": "Used for monitoring PD 99% handle transactions duration in the etcd category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket[5m])) by (instance, result, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1104,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% Peer round trip time seconds",
    "group": "etcd",
    "used_for": "Used for monitoring PD 99% peer round trip time seconds in the etcd category (P99).",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, To, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1105,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% WAL fsync duration",
    "group": "etcd",
    "used_for": "Used for monitoring PD 99% wal fsync duration in the etcd category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1106,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "etcd disk wal fsync rate",
    "group": "etcd",
    "used_for": "Used for monitoring PD etcd disk wal fsync rate in the etcd category. Helps identify disk performance issues.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_disk_wal_fsync_duration_seconds_count{instance=~\".*pd.*\"}[1m]*60)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1107,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99.99% backend commit duration",
    "group": "etcd",
    "used_for": "Used for monitoring PD 99.99% backend commit duration in the etcd category (P9999). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1108,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "etcd disk backend commit rate",
    "group": "etcd",
    "used_for": "Used for monitoring PD etcd disk backend commit rate in the etcd category. Helps identify disk performance issues.",
    "component": "PD",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_disk_backend_commit_duration_seconds_count{instance=~\".*pd.*\"}[1m]*60)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1109,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Raft term",
    "group": "etcd",
    "used_for": "Used for monitoring PD raft term in the etcd category. Helps track Raft consensus performance and replication health.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_server_etcd_state{type=\"term\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1111,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Raft applied index",
    "group": "etcd",
    "used_for": "Used for monitoring PD raft applied index in the etcd category. Helps track Raft consensus performance and replication health.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_server_etcd_state{type=\"appliedIndex\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1110,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Raft committed index",
    "group": "etcd",
    "used_for": "Used for monitoring PD raft committed index in the etcd category. Helps track Raft consensus performance and replication health.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_server_etcd_state{type=\"committedIndex\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1612,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Lease Operations",
    "group": "etcd",
    "used_for": "Used for monitoring PD lease operations in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_lease_granted_total{instance=~\".*pd.*\"}[5m])",
      "rate(etcd_debugging_lease_revoked_total{instance=~\".*pd.*\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1607,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Detailed Backend Commit Durations",
    "group": "etcd",
    "used_for": "Used for monitoring PD detailed backend commit durations in the etcd category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_rebalance_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))",
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_spill_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))",
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_write_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))",
      "rate(etcd_disk_backend_commit_duration_seconds_sum{instance=~\".*pd.*\"}[5m])\n  / rate(etcd_disk_backend_commit_duration_seconds_count{instance=~\".*pd.*\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1608,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Compaction",
    "group": "etcd",
    "used_for": "Used for monitoring PD compaction in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "increase(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum{instance=~\".*pd.*\"}[5m])",
      "increase(etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{instance=~\".*pd.*\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1611,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "WAL and Snapshot",
    "group": "etcd",
    "used_for": "Used for monitoring PD wal and snapshot in the etcd category (P99). Helps track snapshot operations and their impact on performance.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))",
      "histogram_quantile(0.99, sum(rate(etcd_snap_fsync_duration_seconds_bucket{instance=~\".*pd.*\"}[5m])) by (instance, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1614,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC Keys Count",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc keys count in the etcd category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_keys_total{instance=~\".*pd.*\"}[5m]"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1613,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC Operations",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc operations in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_mvcc_put_total{instance=~\".*pd.*\"}[5m])",
      "rate(etcd_debugging_mvcc_range_total{instance=~\".*pd.*\"}[5m])",
      "rate(etcd_debugging_mvcc_delete_total{instance=~\".*pd.*\"}[5m])",
      "rate(etcd_debugging_mvcc_txn_total{instance=~\".*pd.*\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1616,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC Client Operations",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc client operations in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(etcd_server_client_requests_total{instance=~\".*pd.*\"}[5m])) by (client_api_version, type, instance)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1615,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC Total Put Size Growth",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc total put size growth in the etcd category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_mvcc_total_put_size_in_bytes{instance=~\".*pd.*\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1617,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "MVCC Events Counter",
    "group": "etcd",
    "used_for": "Used for monitoring PD mvcc events counter in the etcd category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_delete_total{instance=~\".*pd.*\"}",
      "etcd_debugging_mvcc_events_total{instance=~\".*pd.*\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1618,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Revision",
    "group": "etcd",
    "used_for": "Used for monitoring PD revision in the etcd category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_current_revision{instance=~\".*pd.*\"}",
      "etcd_debugging_mvcc_compact_revision{instance=~\".*pd.*\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1619,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Watcher",
    "group": "etcd",
    "used_for": "Used for monitoring PD watcher in the etcd category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_watcher_total{instance=~\".*pd.*\"}",
      "etcd_debugging_mvcc_watch_stream_total{instance=~\".*pd.*\"}",
      "etcd_debugging_mvcc_slow_watcher_total{instance=~\".*pd.*\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1201,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "PD server TSO handle time",
    "group": "TiDB",
    "used_for": "Used for monitoring TiDB TSO (Timestamp Oracle) pd server tso handle time in the TiDB category (P90/P99/P999/P99999/P9999999999). Helps track timestamp allocation performance.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      99999,
      9999999999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.99999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))",
      "histogram_quantile(0.9999999999, sum(rate(tso_server_handle_tso_duration_seconds_bucket[1m])) by (type, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1202,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "PD client TSO handle time",
    "group": "TiDB",
    "used_for": "Used for monitoring TiDB TSO (Timestamp Oracle) pd client tso handle time in the TiDB category. Helps track timestamp allocation performance.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_client_request_handle_requests_duration_seconds_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1203,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Handle requests count",
    "group": "TiDB",
    "used_for": "Used for monitoring TiDB handle requests count in the TiDB category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tso_server_handle_tso_duration_seconds_count[1m])) by (type)",
      "sum(rate(pd_tso_events[1m])) by (type,dc)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1204,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Tso Request batch size",
    "group": "TiDB",
    "used_for": "Used for monitoring TiDB TSO (Timestamp Oracle) tso request batch size in the TiDB category (P90/P99). Helps track timestamp allocation performance.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [
      90,
      99
    ],
    "variables": [
      "k8s_cluster",
      "tidb_cluster"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket[1m])) by (le))",
      "sum(delta(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(delta(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1301,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Heartbeat region event QPS",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD queries per second (QPS) in the Heartbeat category. Helps track workload volume and throughput changes.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(rate(pd_cluster_region_event{store=~\"$store\"}[1m])) by (address, store, event)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1304,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region heartbeat report",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD region heartbeat report in the Heartbeat category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"report\", status=\"ok\"}[1m])) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1302,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% Region heartbeat handle latency",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD 99% region heartbeat handle latency in the Heartbeat category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [
      99
    ],
    "variables": [
      "store"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{store=~\"$store\"}[1m])) by (address, store, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1303,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "TiKV side heartbeat statistics",
    "group": "Heartbeat",
    "used_for": "Used for monitoring TiKV tikv side heartbeat statistics in the Heartbeat category. Helps track cluster health and communication between components.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tikv_pd_heartbeat_message_total[1m])) by (type)",
      "tikv_pd_pending_heartbeat_total"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1305,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region schedule push",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD region schedule push in the Heartbeat category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(rate(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1306,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region heartbeat report error",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD region heartbeat report error in the Heartbeat category. Helps detect failures and error patterns.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"report\", status=\"err\"}[1m])) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1307,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region heartbeat report active",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD region heartbeat report active in the Heartbeat category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_scheduler_region_heartbeat{store=~\"$store\", type=\"report\", status=\"bind\"}[1m])) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1308,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% store heartbeat handle duration",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD 99% store heartbeat handle duration in the Heartbeat category (P99). Helps identify latency issues and performance degradation.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99
    ],
    "variables": [
      "store"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{store=~\"$store\"}[1m])) by (address, store, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1309,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Heartbeat stream report",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD heartbeat stream report in the Heartbeat category. Helps track cluster health and communication between components.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_hbstream_region_message{store=~\"$store\", type=\"push\", status=\"ok\"}[1m])) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1311,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "99% Bucket Report Interval",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD 99% bucket report interval in the Heartbeat category (P99).",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "store"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{store=~\"$store\"}[1m])) by (address, store, le))"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1312,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Bucket Report State",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD bucket report state in the Heartbeat category.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(rate(pd_server_bucket_report{store=~\"$store\"}[1m])) by (address, store, type,status)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1310,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Heartbeat stream report error",
    "group": "Heartbeat",
    "used_for": "Used for monitoring PD heartbeat stream report error in the Heartbeat category. Helps detect failures and error patterns.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "store"
    ],
    "promql": [
      "sum(delta(pd_hbstream_region_message{store=~\"$store\", type=\"push\", status=\"err\"}[1m])) by (address, store)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1401,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Read Region Byte",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD read region byte in the Heartbeat distribution  category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_read_byte_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1402,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Write Region Byte",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD write region byte in the Heartbeat distribution  category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_write_byte_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1403,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Read Region Key",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD read region key in the Heartbeat distribution  category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_read_key_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1404,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Write Region Key",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD write region key in the Heartbeat distribution  category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_write_key_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1405,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Store Heartbeat Interval",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD store heartbeat interval in the Heartbeat distribution  category. Helps track cluster health and communication between components.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_store_heartbeat_interval_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1406,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Region Heartbeat Interval",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD region heartbeat interval in the Heartbeat distribution  category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_region_heartbeat_interval_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1407,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Bucket Report Interval",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD bucket report interval in the Heartbeat distribution  category.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_server_bucket_report_interval_seconds_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1408,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Hot Degree of Bucket",
    "group": "Heartbeat distribution ",
    "used_for": "Used for monitoring PD hot degree of bucket in the Heartbeat distribution  category.",
    "component": "PD",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(pd_scheduler_buckets_hot_degree_hist_bucket[1m])) by (le)"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1501,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Syncer index",
    "group": "Region storage",
    "used_for": "Used for monitoring PD syncer index in the Region storage category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_region_syncer_status{type=\"sync_index\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1502,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "History last index",
    "group": "Region storage",
    "used_for": "Used for monitoring PD history last index in the Region storage category.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_region_syncer_status{type=\"last_index\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1601,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "State",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD state in the DR Autosync category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "pd_replication_dr_state{instance=\"$instance\"}"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1602,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "State of all instances",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD state of all instances in the DR Autosync category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_replication_dr_state"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1603,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Tick",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD tick in the DR Autosync category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "rate(pd_replication_dr_tick_total{instance=\"$instance\"}[5m])"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1604,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "State ID",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD state id in the DR Autosync category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_replication_dr_state_id"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1605,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Recovered region",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD recovered region in the DR Autosync category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "PD",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_replication_dr_recovered_region"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 1606,
    "dashboard": "Starter-Essential-PD-MS-for-Clinic",
    "title": "Recover progress",
    "group": "DR Autosync",
    "used_for": "Used for monitoring PD recover progress in the DR Autosync category.",
    "component": "PD",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_replication_dr_recover_progress"
    ],
    "dashboard_file": "pd-ms.json"
  },
  {
    "panel_id": 80,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB duration (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_handle_query_duration_seconds_sum{keyspace_id=\"$keyspace_id\", sql_type!=\"internal\"}[2m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{keyspace_id=\"$keyspace_id\", sql_type!=\"internal\"}[2m]))",
      "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type!=\"internal\"}[2m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type!=\"internal\"}[2m])) by (vmrange)))",
      "sum(rate(tidb_server_handle_query_duration_seconds_sum{sql_type!=\"internal\"}[2m])) by (sql_type) / sum(rate(tidb_server_handle_query_duration_seconds_count{sql_type!=\"internal\"}[2m])) by (sql_type)",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\",sql_type!=\"internal\"}[2m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 396,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Database Time by SQL Phase",
    "group": "",
    "used_for": "Used for monitoring TiDB database time by sql phase.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(tidb_server_tokens{keyspace_id=\"$keyspace_id\"})",
      "sum(rate(tidb_session_parse_duration_seconds_sum{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[2m]))",
      "sum(rate(tidb_session_compile_duration_seconds_sum{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[2m]))",
      "sum(rate(tidb_session_execute_duration_seconds_sum{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[2m]))",
      "sum(rate(tidb_server_get_token_duration_seconds_sum{keyspace_id=\"$keyspace_id\"}[2m]))/1000000"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 21,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "QPS",
    "group": "",
    "used_for": "Used for monitoring TiDB queries per second (QPS). Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_executor_statement_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_executor_statement_total{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 395,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Avg TiKV GRPC Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB avg tikv grpc duration. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tikv_grpc_msg_duration_seconds_sum{}[2m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{}[2m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 137,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Failed Query OPM",
    "group": "",
    "used_for": "Used for monitoring TiDB failed query opm. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_execute_error_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 394,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Avg TiDB KV Request Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB avg tidb kv request duration (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_request_seconds_sum{keyspace_id=\"$keyspace_id\", store!=\"0\"}[2m])) by (type)/ sum(rate(tidb_tikvclient_request_seconds_count{keyspace_id=\"$keyspace_id\", store!=\"0\"}[2m])) by (type)",
      "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 112,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Slow query",
    "group": "",
    "used_for": "Used for monitoring TiDB slow query (P90). Helps identify slow operations and their causes.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [
      90
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_slow_query_process_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 42,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Command Per Second",
    "group": "",
    "used_for": "Used for monitoring TiDB command per second.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_query_total{keyspace_id=\"$keyspace_id\"}[1m])) by (result)",
      "sum(rate(tidb_server_query_total{keyspace_id=\"$keyspace_id\", result=\"OK\"}[1m]  offset 1d))",
      "sum(tidb_server_connections{keyspace_id=\"$keyspace_id\"}) * sum(rate(tidb_server_handle_query_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 338,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Request Units (1m sum)",
    "group": "",
    "used_for": "Used for monitoring TiDB request units (1m sum).",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "cluster_id",
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(gateway_connection_flow_bytes_total{namespace=\"tidb-admin\",direction=\"out\",service=~\"gateway-metrics|privatelink-metrics\", cluster_id=\"$cluster_id\"}[1m])) by (cluster_id, service) / 1024",
      "sum(increase(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_sql_layer_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\",type=\"ap\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\",type=\"ap\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(lightning_resource_unit_write_request_unit_sum{namespace=\"import-jobs\",keyspace_id=\"$keyspace_id\"}[1m]))",
      "sum(increase(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(resource_manager_resource_unit_sql_layer_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 390,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "SQL Execute Time Overview",
    "group": "",
    "used_for": "Used for monitoring TiDB sql execute time overview.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_request_seconds_sum{keyspace_id=\"$keyspace_id\", store!=\"0\"}[2m])) by (type)",
      "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{keyspace_id=\"$keyspace_id\", type=\"wait\"}[2m]))",
      "sum(rate(tidb_session_execute_duration_seconds_sum{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[2m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 2,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "CPS By Instance",
    "group": "",
    "used_for": "Used for monitoring TiDB cps by instance.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_server_query_total{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 136,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "999 Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB 999 duration (P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      999
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 267,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Affected Rows By Type",
    "group": "",
    "used_for": "Used for monitoring TiDB affected rows by type.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_affected_rows{keyspace_id=\"$keyspace_id\"}[1m])) by (sql_type)",
      "sum(rate(tidb_server_affected_rows{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 132,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "95 Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB 95 duration (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 218,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Connection Idle Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB connection idle duration (P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "connection",
    "percentile": [
      80,
      90,
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (le,in_txn))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", in_txn='0'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='0'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='0'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='1'}[1m])) by (vmrange, in_txn)))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_conn_idle_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\", in_txn='0'}[1m])) by (vmrange, in_txn)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 134,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "99 Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB 99 duration (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 130,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "80 Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB 80 duration (P80). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le,sql_type))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 23,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration 80 By Instance",
    "group": "",
    "used_for": "Used for monitoring TiDB duration 80 by instance (P80). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 1,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration 95 By Instance",
    "group": "",
    "used_for": "Used for monitoring TiDB duration 95 by instance (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 25,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration 99 By Instance",
    "group": "",
    "used_for": "Used for monitoring TiDB duration 99 by instance (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 81,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration 999 By Instance",
    "group": "",
    "used_for": "Used for monitoring TiDB duration 999 by instance (P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      999
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 339,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Parse Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB parse duration (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 340,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Compile Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB compile duration (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 341,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Execution Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB execution duration (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"general\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 342,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Queries Using Plan Cache OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB queries using plan cache ops. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_plan_cache_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 94,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Failed Query OPM Detail",
    "group": "",
    "used_for": "Used for monitoring TiDB failed query opm detail. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "increase(tidb_server_execute_error_total{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 68,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Internal SQL OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB internal sql ops.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_session_restricted_sql_total{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 272,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Queries in Multi-Statement",
    "group": "",
    "used_for": "Used for monitoring TiDB queries in multi-statement.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_multi_query_num_sum{keyspace_id=\"$keyspace_id\"}[30s]))/sum(rate(tidb_server_multi_query_num_count{keyspace_id=\"$keyspace_id\"}[30s]))",
      "sum(rate(tidb_server_multi_query_num_sum{keyspace_id=\"$keyspace_id\"}[30s]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 391,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Gateway Connections",
    "group": "",
    "used_for": "Used for monitoring TiDB gateway connections. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "cluster_id"
    ],
    "promql": [
      "sum(gateway_connection_total{cluster_id=\"$cluster_id\",stage=\"connect_backend\"}) - sum(gateway_connection_total{cluster_id=\"$cluster_id\",stage=\"close\"})"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 184,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiDB Uptime",
    "group": "",
    "used_for": "Used for monitoring TiDB tidb uptime.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "(time() - process_start_time_seconds{instance=~\"$instance\"})",
      "count(process_start_time_seconds{instance=~\"$instance\"} or vector(0))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 337,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": " TiDB Backend Replicas",
    "group": "",
    "used_for": "Used for monitoring TiDB  tidb backend replicas.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "count(sum(rate(tidb_executor_statement_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 3,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Storage Size",
    "group": "",
    "used_for": "Used for monitoring TiDB storage size. Helps track storage capacity and usage patterns.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "quantile_over_time(0.5, sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"used\", keyspace_id=~\"$keyspace_id\"}))[5m])",
      "quantile_over_time(0.5, sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"tiflash_used\", keyspace_id=~\"$keyspace_id\"}))[5m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 318,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Memory Usage",
    "group": "",
    "used_for": "Used for monitoring TiDB memory usage. Helps identify memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "process_resident_memory_bytes{instance=~\"$instance\"}",
      "go_memory_classes_heap_objects_bytes{instance=~\"$instance\"} + go_memory_classes_heap_unused_bytes{instance=~\"$instance\"} + go_memory_classes_heap_released_bytes{instance=~\"$instance\"} + go_memory_classes_heap_free_bytes{instance=~\"$instance\"}",
      "go_memory_classes_heap_objects_bytes{instance=~\"$instance\"} + go_memory_classes_heap_unused_bytes{instance=~\"$instance\"}",
      "go_memory_classes_heap_objects_bytes{instance=~\"$instance\"}",
      "go_memory_classes_heap_released_bytes{instance=~\"$instance\"} + go_memory_classes_heap_free_bytes{instance=~\"$instance\"}",
      "go_memory_classes_heap_released_bytes{instance=~\"$instance\"}",
      "go_gc_heap_goal_bytes{instance=~\"$instance\"}",
      "tidb_server_memory_usage{keyspace_id=\"$keyspace_id\", job=\"tidb\"}",
      "sum(tidb_server_memory_usage{instance=~\"$instance\"}) by (module, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 393,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Request Units (1m sum)",
    "group": "",
    "used_for": "Used for monitoring TiDB request units (1m sum).",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "cluster_id",
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_sql_layer_request_unit_sum{namespace=\"tidb-serverless\",type=~\"|tp\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(gateway_connection_flow_bytes_total{namespace=\"tidb-admin\",direction=\"out\", cluster_id=\"$cluster_id\"}[1m])) by (cluster_id) / 1024",
      "sum(increase(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\",type=\"ap\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\",type=\"ap\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(increase(lightning_resource_unit_write_request_unit_sum{namespace=\"import-jobs\",keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 378,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "RCU",
    "group": "",
    "used_for": "Used for monitoring TiDB rcu.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(resource_manager_resource_unit_read_request_unit_sum{namespace=\"tidb-serverless\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(rate(resource_manager_resource_unit_write_request_unit_sum{namespace=\"tidb-serverless\", name=\"$keyspace_id\"}[1m])) by(name) + \nsum(rate(resource_manager_resource_unit_sql_layer_request_unit_sum{namespace=\"tidb-serverless\", name=\"$keyspace_id\"}[1m])) by(name)",
      "sum(serverless_metering_rcu_provisioned_capacity{keyspace_id=\"$keyspace_id\"})",
      "sum(serverless_metering_rcu_provisioned_capacity_v2{keyspace_id=\"$keyspace_id\"})",
      "resource_manager_resource_unit_service_limit{namespace=\"tidb-serverless\", name=\"$keyspace_id\"}",
      "resource_manager_resource_unit_available_ru{namespace=\"tidb-serverless\", name=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 168,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "CPU Usage",
    "group": "",
    "used_for": "Used for monitoring TiDB CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "irate(process_cpu_seconds_total{instance=~\"$instance\"}[30s])",
      "tidb_server_maxprocs{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 335,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Go GC Cycles Total",
    "group": "",
    "used_for": "Used for monitoring TiDB go gc cycles total. Helps track garbage collection operations.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(go_gc_cycles_total_gc_cycles_total{container=\"tidb\",instance=~\"$instance\"}[1m])) by (namespace,instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 8,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Connection Count",
    "group": "",
    "used_for": "Used for monitoring TiDB connection count. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_connections{keyspace_id=\"$keyspace_id\"}",
      "sum(tidb_server_connections{keyspace_id=\"$keyspace_id\"})"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 205,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Disconnection Count",
    "group": "",
    "used_for": "Used for monitoring TiDB disconnection count. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_disconnection_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 188,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Open FD Count",
    "group": "",
    "used_for": "Used for monitoring TiDB open fd count.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "process_open_fds{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 61,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Goroutine Count",
    "group": "",
    "used_for": "Used for monitoring TiDB goroutine count.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_sched_goroutines_goroutines{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 49,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Events OPM",
    "group": "",
    "used_for": "Used for monitoring TiDB events opm.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "increase(tidb_server_event_total{keyspace_id=\"$keyspace_id\"}[10m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 82,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Keep Alive OPM",
    "group": "",
    "used_for": "Used for monitoring TiDB keep alive opm.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_monitor_keep_alive_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 165,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Prepare Statement Count",
    "group": "",
    "used_for": "Used for monitoring TiDB prepare statement count.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_prepared_stmts{keyspace_id=\"$keyspace_id\"}",
      "sum(tidb_server_prepared_stmts{keyspace_id=\"$keyspace_id\"})"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 166,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Time Jump Back OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB time jump back ops.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_monitor_time_jump_back_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 54,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Panic And Critial Error",
    "group": "",
    "used_for": "Used for monitoring TiDB panic and critial error. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "increase(tidb_server_panic_total{keyspace_id=\"$keyspace_id\"}[1m])",
      "increase(tidb_server_critical_error_total{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 191,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Skip Binlog Count",
    "group": "",
    "used_for": "Used for monitoring TiDB skip binlog count.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_critical_error_total{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 111,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Get Token Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB get token duration (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_get_token_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_get_token_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 167,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Handshake Error OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB handshake error ops. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_handshake_error_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 211,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Client Data Traffic",
    "group": "",
    "used_for": "Used for monitoring TiDB client data traffic. Helps identify network throughput and bandwidth issues.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_packet_io_bytes{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(tidb_server_packet_io_bytes{keyspace_id=\"$keyspace_id\"}) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 171,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "RCCheckTS WriteConflict Num",
    "group": "",
    "used_for": "Used for monitoring TiDB rccheckts writeconflict num.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_rc_check_ts_conflict_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 382,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Txn File Requests",
    "group": "Txn File",
    "used_for": "Used for monitoring TiDB txn file requests in the Txn File category. Helps track transaction performance and conflict rates.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_txn_file_requests{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 384,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Txn File write bytes",
    "group": "Txn File",
    "used_for": "Used for monitoring TiDB txn file write bytes in the Txn File category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_txn_file_write_bytes{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (scope)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 386,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Txn File mutation size",
    "group": "Txn File",
    "used_for": "Used for monitoring TiDB txn file mutation size in the Txn File category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "sum(delta(tidb_tikvclient_txn_file_mutation_size_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 388,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Txn File duration",
    "group": "Txn File",
    "used_for": "Used for monitoring TiDB txn file duration in the Txn File category (P99/P9999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      99,
      9999
    ],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(delta(tidb_tikvclient_txn_file_duration_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (le,scope))",
      "histogram_quantile(0.99, prometheus_buckets(sum(delta(tidb_tikvclient_txn_file_duration_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, scope)))",
      "sum(rate(tidb_tikvclient_txn_file_duration_sum{instance=~\"$instance\"}[1m])) by (scope) / sum(rate(tidb_tikvclient_txn_file_duration_count{instance=~\"$instance\"}[1m])) by (scope)",
      "histogram_quantile(0.9999, sum(delta(tidb_tikvclient_txn_file_duration_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (le,scope))",
      "histogram_quantile(0.9999, prometheus_buckets(sum(delta(tidb_tikvclient_txn_file_duration_bucket{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}[1m])) by (vmrange, scope)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 69,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction OPS",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction ops in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_session_transaction_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type, txn_mode)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 72,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB duration in the Transaction category (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, txn_mode))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, txn_mode)))",
      "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, txn_mode))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, txn_mode)))",
      "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, txn_mode))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_session_transaction_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, txn_mode)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 74,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Statement Num",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction statement num in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "query",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_session_transaction_statement_num_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 67,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Retry Num",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction retry num in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_session_retry_num_bucket{keyspace_id=~\"$keyspace_id\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 36,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Session Retry Error OPS",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB session retry error ops in the Transaction category. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_session_retry_error_total{keyspace_id=\"$keyspace_id\"}[30s])) by (type, sql_type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 196,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Commit Token Wait Duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB commit token wait duration in the Transaction category (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 4,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Transaction OPS",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB kv transaction ops in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count{keyspace_id=\"$keyspace_id\", type=\"commit\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 193,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Transaction Duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB kv transaction duration in the Transaction category (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 44,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Regions Num",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction regions num in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_txn_regions_num_bucket{instance=~\"$instance\", type=\"2pc_prewrite\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 33,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Write KV Num Rate and Sum",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction write kv num rate and sum in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_tikvclient_txn_write_kv_num_sum{keyspace_id=\"$keyspace_id\"}[30s])",
      "tidb_tikvclient_txn_write_kv_num_sum{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 209,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Write KV Num",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction write kv num in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_txn_write_kv_num_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 201,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Statement Lock Keys",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB statement lock keys in the Transaction category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "query",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_session_statement_lock_keys_count_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 194,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Send HeartBeat Duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB send heartbeat duration in the Transaction category (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_txn_heart_beat_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 34,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Write Size Bytes Rate and Sum",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction write size bytes rate and sum in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_tikvclient_txn_write_size_bytes_sum{keyspace_id=\"$keyspace_id\"}[30s])",
      "tidb_tikvclient_txn_write_size_bytes_sum{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 210,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Write Size Bytes",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction write size bytes in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_txn_write_size_bytes_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 197,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Acquire Pessimistic Locks Duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB acquire pessimistic locks duration in the Transaction category (P80/P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      95,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.80, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_pessimistic_lock_keys_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 195,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Lifetime Reach Counter",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB ttl lifetime reach counter in the Transaction category.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_ttl_lifetime_reach_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 83,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Safepoint OPS",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB load safepoint ops in the Transaction category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_load_safepoint_total{keyspace_id=\"$keyspace_id\", type=\"ok\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 199,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Pessimistic Statement Retry OPS",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB pessimistic statement retry ops in the Transaction category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 212,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Types Per Second",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction types per second in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_commit_txn_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_tikvclient_async_commit_txn_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_tikvclient_one_pc_txn_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 224,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction Commit .99 Backoff",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction commit .99 backoff in the Transaction category (P99). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_count_bucket{keyspace_id=\"$keyspace_id\"}[1m]))",
      "histogram_quantile(0.99, prometheus_buckets(rate(tidb_tikvclient_txn_commit_backoff_count_bucket{keyspace_id=\"$keyspace_id\"}[1m])))",
      "histogram_quantile(0.99, rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m]))",
      "histogram_quantile(0.99, prometheus_buckets(rate(tidb_tikvclient_txn_commit_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 226,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "SafeTS Update Conuter",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB safets update conuter in the Transaction category.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_safets_update_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (result, store)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 225,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Max SafeTS Gap",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB max safets gap in the Transaction category.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_tikvclient_min_safets_gap_seconds{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 231,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Assertion",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB assertion in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(irate(tidb_tikvclient_prewrite_assertion_count{instance=~\"$instance\"}[30s])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 252,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction execution states duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction execution states duration in the Transaction category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.9, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.9, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.8, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.8, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 253,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction with lock execution states duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction with lock execution states duration in the Transaction category (P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [
      80,
      90,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.90, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.80, sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_session_txn_state_seconds_bucket{keyspace_id=\"$keyspace_id\", has_lock=\"true\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 255,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction execution states duration",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction execution states duration in the Transaction category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_session_txn_state_seconds_sum{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_session_transaction_duration_seconds_sum{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 254,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction enter state",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction enter state in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_session_txn_state_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 256,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction leave state",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction leave state in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_session_txn_state_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 261,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Transaction state count change",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB transaction state count change in the Transaction category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_session_txn_state_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type) - on (type) increase(tidb_session_txn_state_entering_count{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 295,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Aggressive Locking Usage",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB aggressive locking usage in the Transaction category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(irate(tidb_session_transaction_aggressive_locking_usage{instance=~\"$instance\"}[30s])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 296,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Aggressive Locking Keys",
    "group": "Transaction",
    "used_for": "Used for monitoring TiDB aggressive locking keys in the Transaction category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(irate(tidb_tikvclient_aggressive_locking_count{instance=~\"$instance\"}[30s])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 156,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Parse Duration 99.99",
    "group": "",
    "used_for": "Used for monitoring TiDB parse duration 99.99 (P99999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99999, sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, sql_type))",
      "histogram_quantile(0.99999, prometheus_buckets(sum(rate(tidb_session_parse_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 154,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Compile Duration 99.99",
    "group": "",
    "used_for": "Used for monitoring TiDB compile duration 99.99 (P9999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, sql_type))",
      "histogram_quantile(0.9999, prometheus_buckets(sum(rate(tidb_session_compile_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 169,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Execution Duration",
    "group": "",
    "used_for": "Used for monitoring TiDB execution duration (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, sql_type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_session_execute_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, sql_type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 76,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Expensive Executors OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB expensive executors ops.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_executor_expensive_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 91,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Queries Using Plan Cache OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB queries using plan cache ops. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_plan_cache_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 258,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Plan Cache Miss OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB plan cache miss ops. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_plan_cache_miss_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 249,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Read From Table Cache OPS",
    "group": "",
    "used_for": "Used for monitoring TiDB read from table cache ops. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_read_from_tablecache_total{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 269,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Plan Cache Memory Usage",
    "group": "",
    "used_for": "Used for monitoring TiDB memory usage. Helps identify memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_plan_cache_instance_memory_usage{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 271,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Plan Cache Plan Num",
    "group": "",
    "used_for": "Used for monitoring TiDB plan cache plan num. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_plan_cache_instance_plan_num_total{keyspace_id=\"$keyspace_id\"}\n"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 12,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Distsql Duration",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB distsql duration in the Distsql category (P50/P90/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      50,
      90,
      99,
      999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_distsql_handle_query_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 14,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Distsql QPS",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Distsql category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_distsql_handle_query_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (copr_type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 60,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Distsql Partial QPS",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Distsql category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_distsql_scan_keys_partial_num_count{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 57,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Scan Keys Num",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB scan keys num in the Distsql category (P50/P90).",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [
      50,
      90
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 58,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Scan Keys Partial Num",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB scan keys partial num in the Distsql category (P80/P90).",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      80,
      90
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_distsql_scan_keys_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 59,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Partial Num",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB partial num in the Distsql category (P50/P90).",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      50,
      90
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_distsql_partial_num_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 41,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Coprocessor Seconds 999",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB coprocessor seconds 999 in the Distsql category (P999). Helps track coprocessor request performance for distributed queries.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [
      999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_tikvclient_cop_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 175,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Coprocessor Cache",
    "group": "Distsql",
    "used_for": "Used for monitoring TiDB coprocessor cache in the Distsql category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_distsql_copr_cache{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 6,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Backoff Duration",
    "group": "KV Errors",
    "used_for": "Used for monitoring TiDB kv backoff duration in the KV Errors category (P80/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      99,
      999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_tikvclient_backoff_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 11,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiClient Region Error OPS",
    "group": "KV Errors",
    "used_for": "Used for monitoring TiDB ticlient region error ops in the KV Errors category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "region",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_region_err_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_tikvclient_region_err_total{keyspace_id=\"$keyspace_id\"}{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 53,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Backoff OPS",
    "group": "KV Errors",
    "used_for": "Used for monitoring TiDB kv backoff ops in the KV Errors category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_backoff_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 32,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Lock Resolve OPS",
    "group": "KV Errors",
    "used_for": "Used for monitoring TiDB lock resolve ops in the KV Errors category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_lock_resolver_actions_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 223,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Replica Selector Failure Per Second",
    "group": "KV Errors",
    "used_for": "Used for monitoring TiDB replica selector failure per second in the KV Errors category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_replica_selector_failure_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 172,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Request OPS",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB kv request ops in the KV Request category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_request_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 48,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Request Duration 99 by store",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB kv request duration 99 by store in the KV Request category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m])) by (le, store))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m])) by (vmrange, store)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 389,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Request Duration 999999 by store and type",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB kv request duration 999999 by store and type in the KV Request category (P999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      999999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999999, sum by(le, store, type) (rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m])))",
      "histogram_quantile(0.999999, prometheus_buckets(sum by(le, store, type) (rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m]))))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 219,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Request Forwarding OPS",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB kv request forwarding ops in the KV Request category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_forward_request_counter{keyspace_id=\"$keyspace_id\"}[1m])) by (from_store, to_store, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 220,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Commit duration heatmap",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB commit duration heatmap in the KV Request category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(delta(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", type=\"Commit\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 30,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "KV Request Duration 99 by type",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB kv request duration 99 by type in the KV Request category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m])) by (le,type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_request_seconds_bucket{keyspace_id=\"$keyspace_id\", store!=\"0\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 250,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Region Cache Error OPS",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB region cache error ops in the KV Request category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_region_cache_operations_total{keyspace_id=\"$keyspace_id\", result=\"err\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 262,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Region Duration",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB load region duration in the KV Request category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_load_region_cache_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "sum(rate(tidb_tikvclient_load_region_cache_seconds_sum{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type) / sum(rate(tidb_tikvclient_load_region_cache_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 164,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Region Cache OK OPS",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB region cache ok ops in the KV Request category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_tikvclient_region_cache_operations_total{keyspace_id=\"$keyspace_id\", result=\"ok\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 263,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "RPC Layer Latency",
    "group": "KV Request",
    "used_for": "Used for monitoring TiDB rpc layer latency in the KV Request category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, store))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, store)))",
      "sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{keyspace_id=\"$keyspace_id\"}[1m])) by (le, store) / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (le, store)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 20,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD Client CMD OPS",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd client cmd ops in the PD Client category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{ type!=\"tso\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 35,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD Client CMD Duration",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd client cmd duration in the PD Client category (P90/P99/P999/P999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      999999999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.90, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.999999999, sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (le, type))",
      "histogram_quantile(0.999999999, prometheus_buckets(sum(rate(tidb_tikvclient_ts_future_wait_seconds_bucket{keyspace_id=\"$keyspace_id\", type!~\"tso|tso_async_wait\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 43,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD Client CMD Fail OPS",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd client cmd fail ops in the PD Client category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 79,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD TSO OPS",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB TSO (Timestamp Oracle) pd tso ops in the PD Client category. Helps track timestamp allocation performance.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\"tso\"}[1m]))",
      "sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\"tso\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 77,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD TSO Wait Duration",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd tso wait duration in the PD Client category (P90/P99/P999/P999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      999999999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))",
      "histogram_quantile(0.999999999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"wait\"}[1m])) by (le))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 78,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD TSO RPC Duration",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd tso rpc duration in the PD Client category (P90/P99/P999/P999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      999999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{instance=~\"$instance\", type=\"tso\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{instance=~\"$instance\", type=\"tso\"}[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{instance=~\"$instance\", type=\"tso\"}[1m])) by (le))",
      "histogram_quantile(0.999999999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{instance=~\"$instance\", type=\"tso\"}[1m])) by (le))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 159,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Async TSO Duration",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB async tso duration in the PD Client category (P90/P99/P999/P999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99,
      999,
      999999999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))",
      "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))",
      "histogram_quantile(0.999999999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\"tso_async_wait\"}[1m])) by (le))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 222,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Request forwarded status",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB request forwarded status in the PD Client category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "pd_client_request_forwarded_status"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 246,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD HTTP Request Duration",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd http request duration in the PD Client category (P90/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      90,
      99,
      999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.999, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.99, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_pd_api_execution_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 247,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD HTTP Request OPS",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd http request ops in the PD Client category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_pd_api_request_total{keyspace_id=\"$keyspace_id\"}[1m]))",
      "sum(rate(tidb_server_pd_api_request_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 248,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "PD HTTP Request Fail OPS",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB pd http request fail ops in the PD Client category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tidb_server_pd_api_request_total{result!~\"200.*\"}[1m]))",
      "sum(rate(tidb_server_pd_api_request_total{result!~\"200.*\"}[1m])) by (type, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 376,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TSO Batch Send Latency",
    "group": "PD Client",
    "used_for": "Used for monitoring TiDB tso batch send latency in the PD Client category (P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      999
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.999, sum(rate(pd_client_request_tso_batch_send_latency_bucket[1m])) by (instance,le))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 27,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Schema Duration",
    "group": "Schema Load",
    "used_for": "Used for monitoring TiDB load schema duration in the Schema Load category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_domain_load_schema_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 251,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Schema Duration",
    "group": "Schema Load",
    "used_for": "Used for monitoring TiDB load schema duration in the Schema Load category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_domain_load_schema_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 29,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Schema Lease Error OPM",
    "group": "Schema Load",
    "used_for": "Used for monitoring TiDB schema lease error opm in the Schema Load category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_session_schema_lease_error_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 28,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Schema OPS",
    "group": "Schema Load",
    "used_for": "Used for monitoring TiDB load schema ops in the Schema Load category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_domain_load_schema_total{keyspace_id=\"$keyspace_id\"}[1m])) by (instance,type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 157,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Load Data From Cached Table Duration",
    "group": "Schema Load",
    "used_for": "Used for monitoring TiDB load data from cached table duration in the Schema Load category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_server_load_table_cache_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_load_table_cache_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 9,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL Duration 95",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl duration 95 in the DDL category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_ddl_handle_job_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 63,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL Backfill Data Rate",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl backfill data rate in the DDL category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_ddl_add_index_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 62,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL Waiting Jobs Count",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl waiting jobs count in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_ddl_waiting_jobs{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 55,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL META OPM",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl meta opm in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "increase(tidb_ddl_worker_operation_total{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 56,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL Worker Duration 99",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl worker duration 99 in the DDL category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type, action, result))",
      "histogram_quantile(0.99, prometheus_buckets(sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type, action, result)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 64,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Deploy Syncer Duration",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB deploy syncer duration in the DDL category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (le, type, result))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (vmrange, type, result)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 65,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Owner Handle Syncer Duration",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB owner handle syncer duration in the DDL category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (le, type, result))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (vmrange, type, result)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 66,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Update Self Version Duration",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB update self version duration in the DDL category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (le, result))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[2m])) by (vmrange, result)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 190,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL OPM",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl opm in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_ddl_handle_job_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_ddl_handle_job_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 192,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL backfill progress in percentage",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl backfill progress in percentage in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_ddl_backfill_percentage_progress{keyspace_id=\"$keyspace_id\"}",
      "tidb_ddl_backfill_percentage_progress{keyspace_id=\"$keyspace_id\", type=\"modify_column\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 264,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL job table duration",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl job table duration in the DDL category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_ddl_job_table_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", }[1m])) by (le, type))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_ddl_job_table_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", }[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 265,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "DDL Running Job Cnt",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB ddl running job cnt in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_ddl_running_job_count{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 266,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Add Index Speed",
    "group": "DDL",
    "used_for": "Used for monitoring TiDB add index speed in the DDL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_ddl_add_index_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 46,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Auto Analyze Duration 95",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB auto analyze duration 95 in the Statistics category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 47,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Auto Analyze QPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Statistics category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_auto_analyze_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 70,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Stats Inaccuracy Rate",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB stats inaccuracy rate in the Statistics category (P50/P90/P99).",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [
      50,
      90,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 71,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Pseudo Estimation OPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB pseudo estimation ops in the Statistics category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_pseudo_estimation_total{keyspace_id=\"$keyspace_id\"}[30s])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 92,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Dump Feedback OPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB dump feedback ops in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_dump_feedback_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 170,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Store Query Feedback QPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Statistics category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_store_query_feedback_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type) "
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 113,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Significant Feedback",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB significant feedback in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_high_error_rate_feedback_total{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 93,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Update Stats OPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB update stats ops in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_update_stats_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 173,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Fast Analyze Status 100",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB fast analyze status 100 in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tidb_statistics_fast_analyze_status_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(1, prometheus_buckets(sum(rate(tidb_statistics_fast_analyze_status_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 229,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Sync Load QPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Statistics category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_sync_load_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)",
      "sum(rate(tidb_statistics_sync_load_timeout_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 230,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Sync Load Latency 95 (ms)",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB sync load latency 95 (ms) in the Statistics category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_statistics_sync_load_latency_millis_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_statistics_sync_load_latency_millis_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.95, sum(rate(tidb_statistics_read_stats_latency_millis_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_statistics_read_stats_latency_millis_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 233,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Stats Healthy Distribution",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB stats healthy distribution in the Statistics category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "avg(tidb_statistics_stats_healthy{keyspace_id=\"$keyspace_id\"}) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 234,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Stats Cache LRU Cost",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB stats cache lru cost in the Statistics category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_statistics_stats_cache_lru_val{keyspace_id=\"$keyspace_id\", type=\"track\"}",
      "tidb_statistics_stats_cache_lru_val{keyspace_id=\"$keyspace_id\", type=\"capacity\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 235,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Stats Cache LRU OPS",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB stats cache lru ops in the Statistics category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_stats_cache_lru_op{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 236,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Plan Replayer Task OPM",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB plan replayer task opm in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_plan_replayer_task{keyspace_id=\"$keyspace_id\", type=~\"dump\"}[1m])) by (result)",
      "sum(rate(tidb_plan_replayer_task{keyspace_id=\"$keyspace_id\", type=~\"capture\"}[1m])) by (result)",
      "avg(tidb_plan_replayer_register_task{keyspace_id=\"$keyspace_id\"})"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 237,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Historical Stats OPM",
    "group": "Statistics",
    "used_for": "Used for monitoring TiDB historical stats opm in the Statistics category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_statistics_historical_stats{keyspace_id=\"$keyspace_id\", type=~\"generate\"}[1m])) by (result)",
      "sum(rate(tidb_statistics_historical_stats{keyspace_id=\"$keyspace_id\", type=~\"dump\"}[1m])) by (result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 162,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "New ETCD Session Duration 95",
    "group": "Owner",
    "used_for": "Used for monitoring TiDB new etcd session duration 95 in the Owner category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance, result))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_owner_new_session_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance, result)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 163,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Owner Watcher OPS",
    "group": "Owner",
    "used_for": "Used for monitoring TiDB owner watcher ops in the Owner category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_owner_watch_owner_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type, result, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 50,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "AutoID QPS",
    "group": "Meta",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the Meta category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_autoid_operation_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m]))",
      "sum(rate(tidb_autoid_operation_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 51,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "AutoID Duration",
    "group": "Meta",
    "used_for": "Used for monitoring TiDB autoid duration in the Meta category (P80/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      99,
      999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))",
      "histogram_quantile(0.999, sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_autoid_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 52,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Meta Operations Duration 99",
    "group": "Meta",
    "used_for": "Used for monitoring TiDB meta operations duration 99 in the Meta category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_meta_operation_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, type)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 85,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Worker Action OPM",
    "group": "GC",
    "used_for": "Used for monitoring TiDB worker action opm in the GC category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_gc_worker_actions_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 86,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Duration 99",
    "group": "GC",
    "used_for": "Used for monitoring TiDB duration 99 in the GC category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_gc_seconds_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 87,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Config",
    "group": "GC",
    "used_for": "Used for monitoring TiDB config in the GC category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "max(tidb_tikvclient_gc_config{keyspace_id=\"$keyspace_id\"}) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 89,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Action Result OPM",
    "group": "GC",
    "used_for": "Used for monitoring TiDB action result opm in the GC category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_gc_action_result{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 181,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Delete Range Task Status",
    "group": "GC",
    "used_for": "Used for monitoring TiDB delete range task status in the GC category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(tidb_tikvclient_range_task_stats{keyspace_id=\"$keyspace_id\"}) by (type, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 90,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Too Many Locks Error OPM",
    "group": "GC",
    "used_for": "Used for monitoring TiDB too many locks error opm in the GC category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_gc_region_too_many_locks{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 158,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Delete Range Failure OPM",
    "group": "GC",
    "used_for": "Used for monitoring TiDB delete range failure opm in the GC category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 88,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GC Failure OPM",
    "group": "GC",
    "used_for": "Used for monitoring TiDB gc failure opm in the GC category. Helps detect failures and error patterns.",
    "component": "TIDB",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_tikvclient_gc_failure{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 182,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Push Task Duration 95",
    "group": "GC",
    "used_for": "Used for monitoring TiDB push task duration 95 in the GC category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_range_task_push_duration_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 203,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "No Available Connection Counter",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB no available connection counter in the Batch Client category. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "delta(tidb_tikvclient_batch_client_no_available_connection_total{keyspace_id=\"$keyspace_id\"}[30s])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 180,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Batch Client Unavailable Duration 95",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB batch client unavailable duration 95 in the Batch Client category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{keyspace_id=\"$keyspace_id\"}[30s])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket{keyspace_id=\"$keyspace_id\"}[30s])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 204,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Wait Connection Establish Duration",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB wait connection establish duration in the Batch Client category (P9999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "connection",
    "percentile": [
      9999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{keyspace_id=\"$keyspace_id\"}[30s])) by (le, instance))",
      "histogram_quantile(0.9999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_client_wait_connection_establish_bucket{keyspace_id=\"$keyspace_id\"}[30s])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 228,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Batch Receive Average Duration",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB batch receive average duration in the Batch Client category (P99/P9999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      9999999999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_tikvclient_batch_recv_latency_sum{keyspace_id=\"$keyspace_id\"}[1m]) / rate(tidb_tikvclient_batch_recv_latency_count{keyspace_id=\"$keyspace_id\"}[1m])",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_batch_recv_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.9999999999, sum(rate(tidb_tikvclient_batch_recv_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.9999999999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_recv_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 343,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Batch Send Average Duration",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB batch send average duration in the Batch Client category (P99/P9999999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      9999999999
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_tikvclient_batch_send_latency_sum{keyspace_id=\"$keyspace_id\"}[1m]) / rate(tidb_tikvclient_batch_send_latency_count{keyspace_id=\"$keyspace_id\"}[1m])",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_send_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.9999999999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (le))",
      "histogram_quantile(0.9999999999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{keyspace_id=\"$keyspace_id\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 344,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Batch Send Requests Count",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB batch send requests count in the Batch Client category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tidb_tikvclient_batch_requests_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 362,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Request Enqueue Duration",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB request enqueue duration in the Batch Client category (P95/P99/P999/P999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      95,
      99,
      999,
      999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.95, prometheus_buckets(sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.999999, sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.999999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_wait_duration_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 364,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Batch Enqueue Duration",
    "group": "Batch Client",
    "used_for": "Used for monitoring TiDB batch enqueue duration in the Batch Client category (P99/P999/P9999/P999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      999,
      9999,
      999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.9999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))",
      "histogram_quantile(0.999999, sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.999999, prometheus_buckets(sum(rate(tidb_tikvclient_batch_send_latency_bucket{instance=~\"$instance\"}[2m])) by (vmrange, instance)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 234,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Ignore Event Per Minute",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB ignore event per minute in the TopSQL category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_topsql_ignored_total{keyspace_id=\"$keyspace_id\"}[1m])) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 239,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "99 Report Duration",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB 99 report duration in the TopSQL category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tidb_topsql_report_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", result=\"ok\"}[5m])) by (le, type))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_topsql_report_duration_seconds_bucket{keyspace_id=\"$keyspace_id\", result=\"ok\"}[5m])) by (vmrange, type)))",
      "histogram_quantile(0.99, sum(rate(tikv_resource_metering_report_duration_seconds_bucket{keyspace_id=\"$keyspace_id\"}[5m])) by (le))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 242,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Report Data Count Per Minute",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB report data count per minute in the TopSQL category.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_topsql_report_data_total_sum{keyspace_id=\"$keyspace_id\"}[1m])) by (type, job)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 243,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Total Report Count",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB total report count in the TopSQL category.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(tidb_topsql_report_duration_seconds_count{keyspace_id=\"$keyspace_id\"}) by (type, result)",
      "sum(increase(tidb_topsql_report_duration_seconds_count{keyspace_id=\"$keyspace_id\"}[1m])) by (type, result)",
      "sum(tikv_resource_metering_report_data_count{keyspace_id=\"$keyspace_id\", type=\"sent\"}) by (type)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 241,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "CPU Profiling OPS",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB CPU utilization in the TopSQL category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tidb_server_cpu_profile_total{keyspace_id=\"$keyspace_id\"}[30s])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 245,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiKV Stat Task OPS",
    "group": "TopSQL",
    "used_for": "Used for monitoring TiDB tikv stat task ops in the TopSQL category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "query",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "rate(tikv_resource_metering_stat_task_count{keyspace_id=\"$keyspace_id\"}[1m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 297,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiDB CPU Usage",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB CPU utilization in the TTL category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "irate(process_cpu_seconds_total{instance=~\"$instance\"}[30s])",
      "tidb_server_maxprocs{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 301,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiKV IO MBps",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB tikv io mbps in the TTL category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "avg(sum(rate(tikv_engine_flow_bytes{keyspace_id=\"$keyspace_id\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
      "max(sum(rate(tikv_engine_flow_bytes{keyspace_id=\"$keyspace_id\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))",
      "max(sum(rate(tikv_engine_flow_bytes{keyspace_id=\"$keyspace_id\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance)) - min(sum(rate(tikv_engine_flow_bytes{keyspace_id=\"$keyspace_id\", db=\"kv\", type=~\"wal_file_bytes|bytes_read|iter_bytes_read\"}[1m])) by (instance))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 299,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TiKV CPU",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB CPU utilization in the TTL category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(process_cpu_seconds_total{keyspace_id=\"$keyspace_id\", job=~\".*tikv\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 279,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL QPS By Type",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB queries per second (QPS) in the TTL category. Helps track workload volume and throughput changes.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_ttl_query_duration_count{keyspace_id=\"$keyspace_id\"}[1m])) by (sql_type, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 302,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Insert Rows Per Second",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl insert rows per second in the TTL category.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_ttl_insert_rows{keyspace_id=\"$keyspace_id\"}[1m]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 287,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Processed Rows Per Second",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl processed rows per second in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_ttl_processed_expired_rows{keyspace_id=\"$keyspace_id\"}[1m])) by (sql_type, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 296,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Insert Rows Per Hour",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl insert rows per hour in the TTL category.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_ttl_insert_rows{keyspace_id=\"$keyspace_id\"}[1h]))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 303,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Delete Rows Per Hour",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl delete rows per hour in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(increase(tidb_server_ttl_processed_expired_rows{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1h])) by (sql_type, result)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 284,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Scan Query Duration",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl scan query duration in the TTL category (P50/P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      50,
      80,
      90,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))\n",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"select\", result=\"ok\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 285,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Delete Query Duration",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl delete query duration in the TTL category (P50/P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "query",
    "percentile": [
      50,
      80,
      90,
      99
    ],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "histogram_quantile(0.50, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.50, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.80, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.80, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.90, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.90, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (vmrange)))",
      "histogram_quantile(0.99, sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (le))",
      "histogram_quantile(0.99, prometheus_buckets(sum(rate(tidb_server_ttl_query_duration_bucket{keyspace_id=\"$keyspace_id\", sql_type=\"delete\", result=\"ok\"}[1m])) by (vmrange)))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 276,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Scan Worker Time By Phase",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB scan worker time by phase in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_ttl_phase_time{keyspace_id=\"$keyspace_id\", type=\"scan_worker\"}[1m])) by (phase)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 282,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Delete Worker Time By Phase",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB delete worker time by phase in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(rate(tidb_server_ttl_phase_time{keyspace_id=\"$keyspace_id\", type=\"delete_worker\"}[1m])) by (phase)\n"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 281,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Job Count By Status",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl job count by status in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(tidb_server_ttl_job_status{keyspace_id=\"$keyspace_id\"}) by (type, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 294,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "TTL Task Count By Status",
    "group": "TTL",
    "used_for": "Used for monitoring TiDB ttl task count by status in the TTL category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "sum(tidb_server_ttl_task_status{keyspace_id=\"$keyspace_id\"}) by (type, instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 289,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GOGC",
    "group": "Resource Manager",
    "used_for": "Used for monitoring TiDB gogc in the Resource Manager category. Helps track garbage collection operations.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "keyspace_id"
    ],
    "promql": [
      "tidb_server_gogc{keyspace_id=\"$keyspace_id\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 293,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "EMA CPU Usage",
    "group": "Resource Manager",
    "used_for": "Used for monitoring TiDB CPU utilization in the Resource Manager category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tidb_rm_ema_cpu_usage{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 346,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Memory Usage",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB memory usage in the Go Runtime category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "process_resident_memory_bytes{instance=~\"$instance\"}",
      "go_memstats_next_gc_bytes{instance=~\"$instance\"} / (1 + tidb_server_gogc{ instance=~\"$instance\"} / 100)",
      "go_memstats_alloc_bytes{instance=~\"$instance\"}",
      "go_memstats_heap_alloc_bytes{instance=~\"$instance\"} - go_memstats_next_gc_bytes{instance=~\"$instance\"} / (1 + tidb_server_gogc{instance=~\"$instance\"} / 100)",
      "go_memstats_heap_idle_bytes{instance=~\"$instance\"} - go_memstats_heap_released_bytes{instance=~\"$instance\"} + go_memstats_heap_inuse_bytes{instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{instance=~\"$instance\"}",
      "go_memstats_stack_sys_bytes{instance=~\"$instance\"} + go_memstats_mspan_sys_bytes{instance=~\"$instance\"} + go_memstats_mcache_sys_bytes{instance=~\"$instance\"} + go_memstats_buck_hash_sys_bytes{instance=~\"$instance\"} + go_memstats_gc_sys_bytes{instance=~\"$instance\"} + go_memstats_other_sys_bytes{instance=~\"$instance\"}",
      "go_memstats_next_gc_bytes{instance=~\"$instance\"}",
      "(clamp_max(idelta(go_memstats_last_gc_time_seconds{instance=~\"$instance\"}[2m]), 1) * go_memstats_next_gc_bytes{instance=~\"$instance\"}) > 0",
      "tidb_server_memory_quota_bytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 366,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "CPU Usage",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB CPU utilization in the Go Runtime category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIDB",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "keyspace_id"
    ],
    "promql": [
      "irate(process_cpu_seconds_total{instance=~\"$instance\"}[30s])",
      "tidb_server_maxprocs{keyspace_id=\"$keyspace_id\", instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 354,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Goroutine Count",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB goroutine count in the Go Runtime category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_goroutines{instance=~\"$instance\"}",
      "go_threads{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 369,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Sched Goroutine Count",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB go sched goroutine count in the Go Runtime category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_sched_goroutines_goroutines{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 350,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Estimated Live Objects",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB estimated live objects in the Go Runtime category.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_memstats_heap_objects{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 360,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "goroutine scheduler latency",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB goroutine scheduler latency in the Go Runtime category (P999/P9999/P99999/P999999). Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      999,
      9999,
      99999,
      999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(go_sched_latencies_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.9999, sum(rate(go_sched_latencies_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.99999, sum(rate(go_sched_latencies_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(0.999999, sum(rate(go_sched_latencies_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))",
      "histogram_quantile(1, sum(rate(go_sched_latencies_seconds_bucket{instance=~\"$instance\"}[2m])) by (le, instance))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 367,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GC Pause Time",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB gc pause time in the Go Runtime category (P99). Helps track garbage collection operations.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum by(instance, le) (rate(go_sched_pauses_total_gc_seconds_bucket{instance=~\"$instance\"}[1m])))",
      "histogram_quantile(1, sum by(instance, le) (rate(go_sched_pauses_total_gc_seconds_bucket{instance=~\"$instance\"}[1m])))"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 356,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "Allocator Throughput",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB allocator throughput in the Go Runtime category.",
    "component": "TIDB",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "irate(go_memstats_alloc_bytes_total{instance=~\"$instance\"}[2m])",
      "irate((go_memstats_alloc_bytes_total{instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{instance=~\"$instance\"})[30s:])",
      "irate(go_memstats_mallocs_total{instance=~\"$instance\"}[2m])",
      "irate(go_memstats_frees_total{instance=~\"$instance\"}[2m])"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 371,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Gc Heap Allocs",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB go gc heap allocs in the Go Runtime category. Helps track garbage collection operations.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(go_gc_heap_allocs_bytes_total{instance=~\"$instance\"}[5m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 352,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GC STW Duration (last 256 GC cycles)",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB gc stw duration (last 256 gc cycles) in the Go Runtime category. Helps identify latency issues and performance degradation.",
    "component": "TIDB",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_gc_duration_seconds{instance=~\"$instance\", quantile=\"0\"}",
      "go_gc_duration_seconds{instance=~\"$instance\", quantile!~\"0|1\"}",
      "go_gc_duration_seconds{instance=~\"$instance\", quantile=\"1\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 372,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Threads",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB go threads in the Go Runtime category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(go_threads{instance=~\"$instance\"}[5m])) by (instance)"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 373,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Memory Classes Heap Objects",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB heap memory allocation in the Go Runtime category. Helps detect memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_memory_classes_heap_objects_bytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 375,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Memory Classes Heap Released",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB heap memory allocation in the Go Runtime category. Helps detect memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_memory_classes_heap_released_bytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 374,
    "dashboard": "TiDB-Serverless-Cluster-Metrics-Clinic",
    "title": "GO Memory Classes Heap Free",
    "group": "Go Runtime",
    "used_for": "Used for monitoring TiDB heap memory allocation in the Go Runtime category. Helps detect memory pressure and potential OOM risks.",
    "component": "TIDB",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "go_memory_classes_heap_free_bytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tidb-cluster.json"
  },
  {
    "panel_id": 235,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Vector Index Build Count by Type",
    "group": "",
    "used_for": "Used for monitoring TiFlash vector index build count by type.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (type, instance) (rate(tiflash_vector_index_build_count{instance=~\"$instance\"}[30s]))\n"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 236,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Vector Index Build Count",
    "group": "",
    "used_for": "Used for monitoring TiFlash vector index build count.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_vector_index_duration_count{instance=~\"$instance\", type=\"build\"}\n"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 239,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Local Index Pool",
    "group": "",
    "used_for": "Used for monitoring TiFlash local index pool.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"LocalIndexPool*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"LocalIndexPool*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance)",
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"LocalIndexSched*\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 243,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "VectorIndex Duration",
    "group": "",
    "used_for": "Used for monitoring TiFlash vectorindex duration (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "histogram_quantile(\n    0.99, \n    sum(rate(\n        tiflash_vector_index_duration_bucket{\n            instance=~\"$instance\",\n            type=\"build\"\n        }\n        [$__rate_interval]\n    )) by (le, type)\n)",
      "histogram_quantile(\n    0.99, \n    sum(rate(\n        tiflash_vector_index_duration_bucket{\n            instance=~\"$instance\",\n        }\n        [$__rate_interval]\n    )) by (le, type)\n)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 231,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "FAP result",
    "group": "FAP",
    "used_for": "Used for monitoring TiFlash fap result in the FAP category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_fap_task_result{instance=~\"$instance\"}) by (type)\n"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 232,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "FAP State",
    "group": "FAP",
    "used_for": "Used for monitoring TiFlash fap state in the FAP category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_fap_task_state{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 237,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "WN CPU Usage",
    "group": "FAP",
    "used_for": "Used for monitoring TiFlash CPU utilization in the FAP category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(tiflash_proxy_process_cpu_seconds_total{instance=~\"tiflash-wn-.*\"}[1m])",
      "sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"tiflash-wn-.*\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 238,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "FAP Duration",
    "group": "FAP",
    "used_for": "Used for monitoring TiFlash fap duration in the FAP category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_fap_task_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))\n"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 53,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Store size",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash store size in the Server category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_StoreSizeUsed{instance=~\"$instance\", type=~\"\"}) by (instance)",
      "sum(tiflash_system_current_metric_StoreSizeUsedRemote{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_StoreSizeUsed{instance=~\"tiflash.*\",keyspace_id=\"\"}) by (instance,type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 54,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Available size",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash available size in the Server category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_StoreSizeAvailable{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 55,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Capacity size",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash capacity size in the Server category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_StoreSizeCapacity{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 21,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Uptime",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash uptime in the Server category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_system_asynchronous_metric_Uptime{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 10,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Memory",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash memory usage in the Server category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIFLASH",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_asynchronous_metric_jemalloc_retained{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_resident{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_active{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{instance=~\"$instance\"})",
      "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{instance=~\"$instance\"})",
      "tiflash_process_rss_by_type_bytes{instance=~\"$instance\", type=\"file\"}",
      "sum(tiflash_process_rss_by_type_bytes{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_MemoryCapacity{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 51,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "CPU Usage",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash CPU utilization in the Server category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "rate(tiflash_proxy_process_cpu_seconds_total{instance=~\"$instance\"}[1m])",
      "sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 213,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash region in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_proxy_tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 200,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Request Unit",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash request unit in the Server category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)",
      "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)",
      "sum(tiflash_system_current_metric_OpenFileForRead{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_OpenFileForReadWrite{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 178,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "IO Throughput",
    "group": "Server",
    "used_for": "Used for monitoring TiFlash io throughput in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{job=\"tiflash\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 141,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "SST Import Service",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash sst import service in the Threads CPU category. Helps track RocksDB storage engine performance.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_importer.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_importer.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 154,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "SST Apply",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash sst apply in the Threads CPU category. Helps track RocksDB storage engine performance.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_low_.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_low_.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 145,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region Task",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash region task in the Threads CPU category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"region_task.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"region_task.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 147,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region Worker",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash region worker in the Threads CPU category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"region_worker.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"region_worker.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 155,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Store",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash raft store in the Threads CPU category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 153,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Snapshot Sender",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash snapshot sender in the Threads CPU category. Helps track snapshot operations and their impact on performance.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"snap_sender.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"snap_sender.*\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 151,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Storage Background (Small Tasks)",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash storage background (small tasks) in the Threads CPU category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"bg_\\\\d+\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"bg_\\\\d+\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 156,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Storage Background (Large Tasks)",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash storage background (large tasks) in the Threads CPU category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"bg_block_\\\\d+\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"bg_block_\\\\d+\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 149,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Manual Compaction",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash manual compaction in the Threads CPU category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"m_compact_pool\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"m_compact_pool\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 159,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "GRPC Async Server",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash grpc async server in the Threads CPU category. Helps track gRPC request performance between components.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"async_poller.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 161,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "GRPC Async Client",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash grpc async client in the Threads CPU category. Helps track gRPC request performance between components.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"GRPCComp.*\"}[1m]))",
      "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 240,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "All other background threads",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash all other background threads in the Threads CPU category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{name=~\"(jemalloc|PDLeader|background|raftstore|apply|grpc|s3).*\"}))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 241,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Vector Index",
    "group": "Threads CPU",
    "used_for": "Used for monitoring TiFlash vector index in the Threads CPU category.",
    "component": "TIFLASH",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{name=~\"LocalIndexPool.*\"}))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 9,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Request QPS",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Coprocessor category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_coprocessor_request_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 2,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Executor QPS",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Coprocessor category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 11,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Request Duration",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash request duration in the Coprocessor category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 12,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Error QPS",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Coprocessor category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 13,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Request Handle Duration",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash request handle duration in the Coprocessor category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 14,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Response Bytes/Seconds",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash response bytes/seconds in the Coprocessor category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_coprocessor_response_bytes{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 63,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Cop task memory usage",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash memory usage in the Coprocessor category (P80/P95/P99/P999). Helps identify memory pressure and potential OOM risks.",
    "component": "TIFLASH",
    "phenomenon": "memory",
    "object": "",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 165,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Exchange Bytes/Seconds",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash exchange bytes/seconds in the Coprocessor category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_exchange_data_bytes{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 100,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Threads of Rpc",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash threads of rpc in the Coprocessor category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_thread_count{instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 77,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Handling Request Number",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash handling request number in the Coprocessor category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_coprocessor_handling_request_count{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 102,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Threads",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash threads in the Coprocessor category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_thread_count{instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 101,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Max Threads of Rpc",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash max threads of rpc in the Coprocessor category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_thread_count{instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 157,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "MPP Query count",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash mpp query count in the Coprocessor category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "query",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_mpp_task_manager{instance=~\"$instance\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 103,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Max Threads",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash max threads in the Coprocessor category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_thread_count{instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 166,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Data size in send and receive queue",
    "group": "Coprocessor",
    "used_for": "Used for monitoring TiFlash data size in send and receive queue in the Coprocessor category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_exchange_queueing_data_bytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 107,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Min TSO",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash TSO (Timestamp Oracle) min tso in the Task Scheduler category. Helps track timestamp allocation performance.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"min_tso\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 109,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Estimated Thread Usage and Limit",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash estimated thread usage and limit in the Task Scheduler category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"thread_soft_limit\"}) by (instance, type)",
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"estimated_thread_usage\"}) by (instance, type)",
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"thread_hard_limit\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 111,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Active and Waiting Queries Count",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash active and waiting queries count in the Task Scheduler category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"waiting_queries_count\"}) by (instance, type)",
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"active_queries_count\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 113,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Active and Waiting Tasks Count",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash active and waiting tasks count in the Task Scheduler category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"waiting_tasks_count\"}) by (instance, type)",
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"active_tasks_count\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 117,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Hard Limit Exceeded Count",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash hard limit exceeded count in the Task Scheduler category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_task_scheduler{instance=~\"$instance\", type=\"hard_limit_exceeded_count\"}) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 115,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Task Waiting Duration",
    "group": "Task Scheduler",
    "used_for": "Used for monitoring TiFlash task waiting duration in the Task Scheduler category (P80/P90). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      90
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (instance,le))",
      "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (instance,le))",
      "histogram_quantile(1, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (instance,le))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 17,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Schema Version",
    "group": "DDL",
    "used_for": "Used for monitoring TiFlash schema version in the DDL category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tiflash_schema_version{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 18,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Schema Apply OPM",
    "group": "DDL",
    "used_for": "Used for monitoring TiFlash schema apply opm in the DDL category. Helps track Raft apply operations performance.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(increase(tiflash_schema_apply_count{instance=~\"$instance\"}[1m])) by (type)",
      "avg(increase(tiflash_schema_trigger_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 19,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Schema Internal DDL OPM",
    "group": "DDL",
    "used_for": "Used for monitoring TiFlash schema internal ddl opm in the DDL category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(increase(tiflash_schema_internal_ddl_count{instance=~\"$instance\"}[1m])) by (type)",
      "sum(increase(tiflash_schema_internal_ddl_count{instance=~\"$instance\"}[1m]))",
      "sum(increase(tiflash_schema_internal_ddl_count{instance=~\"$instance\"}[1m])) by (type,instance)",
      "sum(increase(tiflash_schema_internal_ddl_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 20,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Schema Apply Duration",
    "group": "DDL",
    "used_for": "Used for monitoring TiFlash schema apply duration in the DDL category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(tiflash_schema_applying{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 41,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write Command OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash write command ops in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tiflash_storage_command_count{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_DMWriteBlock{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 38,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write Amplification",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash write amplification in the Storage category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}\n)",
      "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)",
      "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}[10m])\n)",
      "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}[30m])\n)",
      "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}[5m])\n)",
      "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 40,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Read Tasks OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash read tasks ops in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_read_tasks_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 39,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Small Internal Tasks OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash small internal tasks ops in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_subtask_count{instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 42,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Small Internal Tasks Duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash small internal tasks duration in the Storage category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le,type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 130,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Large Internal Tasks OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash large internal tasks ops in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_subtask_count{instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 131,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Large Internal Tasks Duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash large internal tasks duration in the Storage category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (le,type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 205,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Opened File Count",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash opened file count in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_proxy_process_open_fds{job=\"tiflash\"}",
      "sum(tiflash_system_current_metric_OpenFileForWrite{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_OpenFileForRead{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_OpenFileForReadWrite{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 206,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "File Open OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash file open ops in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_FileOpen{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tiflash_system_profile_event_FileOpenFailed{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 207,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "FSync Status",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash fsync status in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_FileFSync{instance=~\"$instance\"}[1m])) by (instance)",
      "histogram_quantile(1, sum(rate(tiflash_system_seconds_bucket{instance=~\"$instance\", type=~\"fsync\"}[$__rate_interval])) by (le, instance))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 46,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Disk Write OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash disk write ops in the Storage category. Helps identify disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_PSMWritePages{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 47,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Disk Read OPS",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash disk read ops in the Storage category. Helps identify disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_PSMReadPages{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_ReadBufferAIORead{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 175,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write flow",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash write flow in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_PSMWriteBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 176,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Read flow",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash read flow in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_PSMReadBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{instance=~\"$instance\"}[1m]))",
      "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 67,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Current Data Management Tasks",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash current data management tasks in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tiflash_system_current_metric_DT_DeltaMerge{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_DT_SegmentSplit{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_DT_SegmentMerge{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 214,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "DeltaIndexError",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash deltaindexerror in the Storage category. Helps detect failures and error patterns.",
    "component": "TIFLASH",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "k8s_cluster",
      "tidb_cluster"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 84,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "I/O Limiter",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash i/o limiter in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_io_limiter{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 86,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "I/O Limiter Pending Tasks",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash i/o limiter pending tasks in the Storage category (P99).",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{instance=~\"$instance\"}) by (instance)",
      "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{instance=~\"$instance\"}) by (instance)",
      "histogram_quantile(1.00, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 132,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Read Thread and Data Sharing",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash read thread and data sharing in the Storage category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_read_thread_counter{instance=~\"$instance\", type!~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy|sche_no_segment\"}[1m])) by (type)",
      "sum(rate(tiflash_storage_read_thread_counter{instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[1m]))",
      "sum(rate(tiflash_storage_read_thread_counter{instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[1m]))",
      "sum(rate(tiflash_storage_read_thread_counter{instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[1m]))/sum(rate(tiflash_storage_read_thread_counter{instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 88,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Read Snapshots",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash read snapshots in the Storage category. Helps track snapshot operations and their impact on performance.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_system_current_metric_DT_SegmentReadTasks{instance=~\"$instance\"}",
      "tiflash_system_current_metric_PSMVCCSnapshotsList{instance=~\"$instance\"}",
      "tiflash_system_current_metric_PSMVCCNumSnapshots{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfRead{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfReadRaw{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{instance=~\"$instance\"}",
      "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{instance=~\"$instance\"}",
      "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{instance=~\"$instance\"}",
      "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{instance=~\"$instance\"}",
      "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 208,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Effectiveness of Mark Cache",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash effectiveness of mark cache in the Storage category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_system_profile_event_MarkCacheMisses{instance=~\"$instance\"}) by (instance)",
      "max(tiflash_system_profile_event_MarkCacheHits{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 209,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Schema of Column File",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash schema of column file in the Storage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tiflash_shared_block_schemas{instance=~\"$instance\", type=~\"current_size\"}) by (instance)",
      "sum(rate(tiflash_shared_block_schemas{instance=~\"$instance\", type=~\"hit_count\"}[1m])) by (instance)",
      "max(tiflash_shared_block_schemas{instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)",
      "max(tiflash_shared_block_schemas{instance=~\"$instance\", type=~\"miss_count\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 215,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Effectiveness of Mark Cache",
    "group": "Storage",
    "used_for": "Used for monitoring TiFlash effectiveness of mark cache in the Storage category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "k8s_cluster",
      "tidb_cluster"
    ],
    "promql": [
      "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
      "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
      "tiflash_system_asynchronous_metric_RNDeltaIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 62,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write Stall Duration",
    "group": "Storage Write Stall",
    "used_for": "Used for monitoring TiFlash write stall duration in the Storage Write Stall category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, instance))",
      "histogram_quantile(1, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, instance))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 210,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write & Delta Management Throughput",
    "group": "Storage Write Stall",
    "used_for": "Used for monitoring TiFlash write & delta management throughput in the Storage Write Stall category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"}[1m]))",
      "sum(rate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type!~\"write|ingest\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 211,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write & Delta Management Total",
    "group": "Storage Write Stall",
    "used_for": "Used for monitoring TiFlash write & delta management total in the Storage Write Stall category.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write|ingest\"})",
      "sum(tiflash_storage_throughput_bytes{instance=~\"$instance\", type!~\"write|ingest\"})"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 90,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write Throughput By Instance",
    "group": "Storage Write Stall",
    "used_for": "Used for monitoring TiFlash write throughput by instance in the Storage Write Stall category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"write\"}[1m])) by (instance)",
      "sum(rate(tiflash_storage_throughput_bytes{instance=~\"$instance\", type=~\"ingest\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 212,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Write Command OPS By Instance",
    "group": "Storage Write Stall",
    "used_for": "Used for monitoring TiFlash write command ops by instance in the Storage Write Stall category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_DMWriteBlock{instance=~\"$instance\"}[1m])) by (instance, type)",
      "sum(increase(tiflash_storage_command_count{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 128,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PageStorage Disk Usage",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash pagestorage disk usage in the PageStorage category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_system_asynchronous_metric_BlobDiskBytes{instance=~\"$instance\"}",
      "sum(tiflash_system_asynchronous_metric_BlobValidBytes{instance=~\"$instance\"}) by (instance)",
      "sum((tiflash_system_asynchronous_metric_BlobValidBytes{instance=~\"$instance\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{instance=~\"$instance\"})) by (instance)",
      "tiflash_system_asynchronous_metric_LogDiskBytes{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 129,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PageStorage File Num",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash pagestorage file num in the PageStorage category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_asynchronous_metric_BlobFileNums{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_asynchronous_metric_LogNums{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 133,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PageStorage WriteBatch Size",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash pagestorage writebatch size in the PageStorage category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_storage_page_write_batch_size_bucket{instance=~\"$instance\", type=\"v3\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 158,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Page write Duration",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash page write duration in the PageStorage category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1.00, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 163,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Page GC Tasks OPM",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash page gc tasks opm in the PageStorage category. Helps track garbage collection operations.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tiflash_storage_page_gc_count{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 162,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Page GC Duration",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash page gc duration in the PageStorage category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 164,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Numer of Pages",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash numer of pages in the PageStorage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tiflash_system_asynchronous_metric_PagesInMem{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 123,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Number of Tables",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash number of tables in the PageStorage category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "k8s_cluster",
      "tidb_cluster"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_StoragePoolV2Only{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_StoragePoolV3Only{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_StoragePoolMixMode{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 197,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PageStorage stored bytes by type",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash pagestorage stored bytes by type in the PageStorage category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_storage_page_data_by_types{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 216,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PageStorage Pending Writers Num",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash pagestorage pending writers num in the PageStorage category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_PSPendingWriterNum{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 217,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "PS Command OPS By Instance",
    "group": "PageStorage",
    "used_for": "Used for monitoring TiFlash ps command ops by instance in the PageStorage category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_page_command_count{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 166,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Stale Read OPS",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash stale read ops in the Raft category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_stale_read_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 35,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Read Index OPS",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft read index ops in the Raft category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_read_index_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 37,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Wait Index Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft wait index duration in the Raft category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 36,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Batch Read Index Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft batch read index duration in the Raft category (P80/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 82,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Apply Raft write logs Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash apply raft write logs duration in the Raft category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[1m])) ",
      "histogram_quantile(1.00, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{instance=~\"$instance\", type=\"admin\"}[1m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{instance=~\"$instance\", type=\"admin\"}[1m])) "
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 218,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Heavy Raft Apply Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash heavy raft apply duration in the Raft category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance",
      "k8s_cluster",
      "tidb_cluster"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 76,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region write Duration (decode)",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash region write duration (decode) in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{instance=~\"$instance\", type=\"decode\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 87,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region write Duration (write blocks)",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash region write duration (write blocks) in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 81,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Apply Raft write logs Duration [Heatmap]",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash apply raft write logs duration [heatmap] in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 93,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Apply Raft admin logs Duration [Heatmap]",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash apply raft admin logs duration [heatmap] in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{instance=~\"$instance\", type=\"admin\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 220,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Events QPS",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Raft category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_raft_events_count{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 221,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Frequent Events QPS",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Raft category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_raft_frequent_events_count{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 223,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Log Gap Heatmap",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft log gap heatmap in the Raft category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_raft_log_gap_count_bucket{instance=~\"$instance\", type=~\"applied_index\"}[1m])) by (le, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 224,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Entry Batch Size Heatmap",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft entry batch size heatmap in the Raft category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_entry_size_bucket{instance=~\"$instance\", type=~\"normal\"}[1m])) by (le, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 222,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Region Size Heatmap",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash region size heatmap in the Raft category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_region_flush_bytes_bucket{instance=~\"$instance\", type=~\"unflushed\"}[1m])) by (le, type)",
      "sum(delta(tiflash_raft_region_flush_bytes_bucket{instance=~\"$instance\", type=~\"flushed\"}[1m])) by (le, type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 225,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Ongoing raft snapshot",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash ongoing raft snapshot in the Raft category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 226,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Eager GC OPS",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft eager gc ops in the Raft category. Helps track Raft consensus performance and replication health.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_eager_gc_count{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 227,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Raft Eager GC Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash raft eager gc duration in the Raft category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 75,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Applying snapshots Count",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash applying snapshots count in the Raft category. Helps track snapshot operations and their impact on performance.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 219,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Keys flow",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash keys flow in the Raft category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_raft_process_keys{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 72,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Snapshot Predecode Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash snapshot predecode duration in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot_predecode\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 127,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Snapshot Predecode SST to DT Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash snapshot predecode sst to dt duration in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot_predecode_sst2dt\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 73,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Snapshot Flush Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash snapshot flush duration in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot_flush\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 74,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Ingest SST Duration",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash ingest sst duration in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\"$instance\", type=\"ingest_sst\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 92,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Upstream Latency [Heatmap]",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash upstream latency [heatmap] in the Raft category. Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_raft_upstream_latency_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 91,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Upstream Latency",
    "group": "Raft",
    "used_for": "Used for monitoring TiFlash upstream latency in the Raft category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{instance=~\"$instance\"}[1m]))) by (le) / 1000000000)",
      "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tiflash_raft_upstream_latency_sum{instance=~\"$instance\"}[1m])) / sum(rate(tiflash_raft_upstream_latency_count{instance=~\"$instance\"}[1m])) "
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 99,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Rough Set Filter Rate",
    "group": "Rough Set Filter Rate Histogram",
    "used_for": "Used for monitoring TiFlash rough set filter rate in the Rough Set Filter Rate Histogram category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\"$instance\"}[1m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\"$instance\"}[1m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\"$instance\"}[1m]))) by (instance)",
      "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\"$instance\"}[5m]))) by (instance)",
      "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 97,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Rough Set Filter Rate Histogram",
    "group": "Rough Set Filter Rate Histogram",
    "used_for": "Used for monitoring TiFlash rough set filter rate histogram in the Rough Set Filter Rate Histogram category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 184,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Read Duration Breakdown",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash read duration breakdown in the Disaggregated category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 189,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Checkpoint Upload Duration",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash checkpoint upload duration in the Disaggregated category (P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      90,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1.00, sum(rate(tiflash_storage_checkpoint_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, sum(rate(tiflash_storage_checkpoint_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_storage_checkpoint_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 194,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Checkpoint Upload flow",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash checkpoint upload flow in the Disaggregated category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_checkpoint_flow{instance=~\"$instance\", type=\"incremental\"}[1m]))",
      "sum(rate(tiflash_storage_checkpoint_flow{instance=~\"$instance\", type=\"compaction\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 187,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Checkpoint Upload keys by type (all)",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash checkpoint upload keys by type (all) in the Disaggregated category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_checkpoint_keys_by_types{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 190,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Checkpoint Upload flow by type (incremental+compaction)",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash checkpoint upload flow by type (incremental+compaction) in the Disaggregated category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_checkpoint_flow_by_types{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 192,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote File Num",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash remote file num in the Disaggregated category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_storage_remote_stats{instance=~\"$instance\", type=\"num_files\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 188,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Store Usage",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash remote store usage in the Disaggregated category. Helps track storage capacity and usage patterns.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_storage_remote_stats{instance=~\"$instance\", type=\"total_size\"}) by (instance)",
      "sum(tiflash_storage_remote_stats{instance=~\"$instance\", type=\"valid_size\"}) by (instance)",
      "sum((tiflash_storage_remote_stats{instance=~\"$instance\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{instance=~\"$instance\", type=\"total_size\"})) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 191,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Object Lock Request QPS",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash queries per second (QPS) in the Disaggregated category. Helps track workload volume and throughput changes.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_disaggregated_object_lock_request_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 186,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Object Lock Duration",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash remote object lock duration in the Disaggregated category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 185,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote GC Duration Breakdown",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash remote gc duration breakdown in the Disaggregated category (P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      90,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_gc_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 193,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote GC Status",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash remote gc status in the Disaggregated category. Helps track garbage collection operations.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_storage_s3_gc_status{instance=~\"$instance\"}) by (instance,type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 201,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Request Unit",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash request unit in the Disaggregated category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)",
      "sum(increase(tiflash_storage_sync_replica_ru[24h])) by (keyspace_id)",
      "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)",
      "sum(increase(tiflash_compute_request_unit[24h])) by (cluster_id)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 202,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Memory Usage of Storage Tasks",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash memory usage in the Disaggregated category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIFLASH",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 228,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "DeltaIndexCache",
    "group": "Disaggregated",
    "used_for": "Used for monitoring TiFlash deltaindexcache in the Disaggregated category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_delta_index_cache{instance=~\"$instance\"}[1m])) by (type, instance)",
      "sum(rate(tiflash_storage_delta_index_cache{instance=~\"$instance\", type=~\"hit\"}[1m])) by (instance) /sum(rate(tiflash_storage_delta_index_cache{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 199,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "S3 Bytes",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash s3 bytes in the S3 category.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_S3WriteBytes{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3ReadBytes{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 181,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "S3 OPS",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash s3 ops in the S3 category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "k8s_cluster"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_S3PutObject{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3GetObject{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3HeadObject{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3ListObjects{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 180,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "S3 Retry OPS",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash s3 retry ops in the S3 category.",
    "component": "TIFLASH",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 179,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "S3 Request Duration",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash s3 request duration in the S3 category (P80/P90/P99). Helps identify latency issues and performance degradation.",
    "component": "TIFLASH",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      80,
      90,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1.0, sum(rate(tiflash_storage_s3_request_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 198,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Cache Operations",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash remote cache operations in the S3 category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIFLASH",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_remote_cache{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tiflash_storage_remote_cache{instance=~\"$instance\", type=~\"dtfile_hit\"}[1m]))/sum(rate(tiflash_storage_remote_cache{instance=~\"$instance\", type=~\"dtfile_hit|dtfile_miss\"}[1m]))",
      "sum(rate(tiflash_storage_remote_cache{instance=~\"$instance\", type=~\"page_hit\"}[1m]))/sum(rate(tiflash_storage_remote_cache{instance=~\"$instance\", type=~\"page_hit|page_miss\"}[1m]))"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 204,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Cache Flow",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash remote cache flow in the S3 category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tiflash_storage_remote_cache_bytes{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 203,
    "dashboard": "Starter-Essential-TiFlash-Summary-for-Clinic",
    "title": "Remote Cache Usage",
    "group": "S3",
    "used_for": "Used for monitoring TiFlash remote cache usage in the S3 category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIFLASH",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tiflash_system_current_metric_DTFileCacheCapacity{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_DTFileCacheUsed{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_PageCacheCapacity{instance=~\"$instance\"}) by (instance)",
      "sum(tiflash_system_current_metric_PageCacheUsed{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tiflash-summary.json"
  },
  {
    "panel_id": 12842,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write Pipeline Duration",
    "group": "Duration",
    "used_for": "Used for monitoring TiKV write pipeline duration in the Duration category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12970,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Cop Read Duration",
    "group": "Duration",
    "used_for": "Used for monitoring TiKV cop read duration in the Duration category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 56,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store size",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV store size in the Cluster category. Helps track storage capacity and usage patterns.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"all_used\"}) by (instance)",
      "sum(tikv_store_size_bytes{instance=~\".*canary.*\", type=\"all_used\"})",
      "sum(tikv_store_size_bytes{instance=~\".*canary.*\", type=\"available\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1706,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Available size",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV available size in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"available\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1707,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Capacity size",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV capacity size in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"capacity\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1708,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "CPU",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV CPU utilization in the Cluster category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(process_cpu_seconds_total{instance=~\"$instance\", job=~\".*tikv-serverless.*\"}[1m])) by (instance,job)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1709,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memory",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV memory usage in the Cluster category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(process_resident_memory_bytes{job=~\".*tikv.*\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1710,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "IO utilization",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV io utilization in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(node_disk_io_time_seconds_total{}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1711,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "MBps",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV mbps in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[1m])) by (instance)",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1713,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "QPS",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV queries per second (QPS) in the Cluster category. Helps track workload volume and throughput changes.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance,type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1712,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Errps",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV errps in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance)",
      "sum(delta(tikv_pd_heartbeat_message_total{instance=~\"$instance\", type=\"noop\"}[1m])) by (instance) < 1",
      "sum(rate(tikv_critical_error_total{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1715,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Leader",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV leader in the Cluster category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}) by (instance)",
      "delta(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}[30s]) < -10"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1714,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV region in the Cluster category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)",
      "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"buckets\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4106,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Uptime",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV uptime in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "(time() - process_start_time_seconds{job=~\".*tikv-serverless.*\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763574964,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Dfs Worker Healthy",
    "group": "Cluster",
    "used_for": "Used for monitoring TiKV dfs worker healthy in the Cluster category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "raft_engine_dfs_worker_healthy{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2741,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Critical error",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV critical error in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_critical_error_total{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1584,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Server is busy",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV server is busy in the Errors category.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tikv_channel_full_total{instance=~\"$instance\"}[1m])) by (instance, type)",
      "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\", type='full'}[1m])) by (instance)",
      "avg(tikv_engine_write_stall{instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)",
      "sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 18,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Server report failures",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV server report failures in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_server_report_failure_msg_total{instance=~\"$instance\"}[1m])) by (type,instance,store_id)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1718,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raftstore error",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV raftstore error in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_engine_async_request_total{instance=~\"$instance\", status!~\"success|all\"}[1m])) by (instance, status)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1719,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler error",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV scheduler error in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[1m])) by (instance, stage)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1720,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Coprocessor error",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV coprocessor error in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (instance, reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1721,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC message error",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV grpc message error in the Errors category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1722,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Leader drop",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV leader drop in the Errors category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1723,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Leader missing",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV leader missing in the Errors category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_leader_missing{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572510,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Damaged files",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV damaged files in the Errors category.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_rocksdb_damaged_files{}",
      "tikv_rocksdb_damaged_files_deleted{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572588,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Log Replication Rejected",
    "group": "Errors",
    "used_for": "Used for monitoring TiKV log replication rejected in the Errors category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_server_raft_append_rejects{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 33,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "CF size",
    "group": "Server",
    "used_for": "Used for monitoring TiKV cf size in the Server category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_engine_size_bytes{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 22,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Channel full",
    "group": "Server",
    "used_for": "Used for monitoring TiKV channel full in the Server category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_channel_full_total{instance=~\"$instance\"}[1m])) by (instance, type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 75,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Active written leaders",
    "group": "Server",
    "used_for": "Used for monitoring TiKV active written leaders in the Server category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_region_written_keys_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1481,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Approximate Region size",
    "group": "Server",
    "used_for": "Used for monitoring TiKV approximate region size in the Server category (P95/P99). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_raftstore_region_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_raftstore_region_size_count{instance=~\"$instance\"}[1m])) "
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3638,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Approximate Region size Histogram",
    "group": "Server",
    "used_for": "Used for monitoring TiKV approximate region size histogram in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 58,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region average written bytes",
    "group": "Server",
    "used_for": "Used for monitoring TiKV region average written bytes in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tikv_region_written_bytes_sum{}[1m])) by (instance) / sum(rate(tikv_region_written_bytes_count{}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3646,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region written bytes",
    "group": "Server",
    "used_for": "Used for monitoring TiKV region written bytes in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tikv_region_written_bytes_bucket[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 57,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region average written keys",
    "group": "Server",
    "used_for": "Used for monitoring TiKV region average written keys in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_region_written_keys_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_region_written_keys_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3647,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region written keys",
    "group": "Server",
    "used_for": "Used for monitoring TiKV region written keys in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_region_written_keys_bucket{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3720,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request batch input",
    "group": "Server",
    "used_for": "Used for monitoring TiKV request batch input in the Server category (P99).",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_server_request_batch_size_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_server_request_batch_size_count{instance=~\"$instance\"}[1m])) by (type)",
      "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3730,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Hibernate Peers",
    "group": "Server",
    "used_for": "Used for monitoring TiKV hibernate peers in the Server category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_hibernated_peer_state{instance=~\"$instance\"}) by (instance, state)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7266,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memory trace",
    "group": "Server",
    "used_for": "Used for monitoring TiKV memory usage in the Server category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_server_mem_trace_sum{instance=~\"$instance\", name=~\"raftstore-.*\"}",
      "raft_engine_memory_usage{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9560,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft Entry Cache Evicts",
    "group": "Server",
    "used_for": "Used for monitoring TiKV raft entry cache evicts in the Server category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "rate(tikv_raft_entries_evict_bytes{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12971,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Resolve address duration",
    "group": "Server",
    "used_for": "Used for monitoring TiKV resolve address duration in the Server category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_server_address_resolve_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572581,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Thread Pool Schedule Wait Duration",
    "group": "Server",
    "used_for": "Used for monitoring TiKV 99% thread pool schedule wait duration in the Server category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\"}[1m])) by (le, name))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572692,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Average Thread Pool Schedule Wait Duration",
    "group": "Server",
    "used_for": "Used for monitoring TiKV average thread pool schedule wait duration in the Server category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_yatp_pool_schedule_wait_duration_sum{instance=~\"$instance\"}[1m])) by (name) /  sum(rate(tikv_yatp_pool_schedule_wait_duration_count{instance=~\"$instance\"}[1m])) by (name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572784,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Disk IO time per second",
    "group": "Server",
    "used_for": "Used for monitoring TiKV disk io time per second in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_time\"}[1m])) by (req)",
      "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_time\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572785,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Disk IO bytes per second",
    "group": "Server",
    "used_for": "Used for monitoring TiKV disk io bytes per second in the Server category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_byte\"}[1m])) by (req)",
      "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_byte\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 95,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC message count",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV grpc message count in the gRPC category. Helps track gRPC request performance between components.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 107,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC message failed",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV grpc message failed in the gRPC category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2532,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Average gRPC message duration",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV average grpc message duration in the gRPC category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_msg_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2533,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC batch size",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV grpc batch size in the gRPC category (P99). Helps track gRPC request performance between components.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_server_grpc_req_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_grpc_req_batch_size_count{instance=~\"$instance\"}[1m]))",
      "sum(rate(tikv_server_grpc_resp_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_server_request_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_request_batch_size_count{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573634,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% gRPC message duration",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV 99% grpc message duration in the gRPC category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2534,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "raft message batch size",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV raft message batch size in the gRPC category (P99). Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_server_raft_message_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_raft_message_batch_size_count{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 98,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "9999% gRPC message duration",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV 9999% grpc message duration in the gRPC category (P9999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572859,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC request sources duration",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV grpc request sources duration in the gRPC category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_request_source_duration_vec{instance=~\"$instance\"}[1m])) by (source)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573635,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "9999999999% gRPC message duration",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV 9999999999% grpc message duration in the gRPC category (P9999999999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      9999999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999999999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572858,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC request sources QPS",
    "group": "gRPC",
    "used_for": "Used for monitoring TiKV queries per second (QPS) in the gRPC category. Helps track workload volume and throughput changes.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_grpc_request_source_counter_vec{instance=~\"$instance\"}[1m])) by (source)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 61,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft store CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 79,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Async apply CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_[0-9]+\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13115,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store writer CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_write.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 105,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "gRPC poll CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"grpc.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 64,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sched_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1908,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage ReadPool CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_norm.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_high.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_low.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4287,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Unified read pool CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"unified_read_po.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 69,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "RocksDB CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 78,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Coprocessor CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_normal.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_high.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_low.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2531,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 68,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "BackGround Worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 692,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0",
      "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 691,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_endpoint\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance, tid) > 0",
      "sum(tikv_backup_thread_pool_size{instance=~\"$instance\"}) by(instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 62,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "CDC worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"tso\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 60,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "CDC endpoint CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cdc_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572511,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raftlog fetch Worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftlog_fetch.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9962,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TSO Worker CPU",
    "group": "Thread CPU",
    "used_for": "Used for monitoring TiKV CPU utilization in the Thread CPU category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"tso_worker\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6985,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TTL check progress",
    "group": "TTL",
    "used_for": "Used for monitoring TiKV ttl check progress in the TTL category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_ttl_checker_processed_regions{instance=~\"$instance\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6987,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TTL checker actions",
    "group": "TTL",
    "used_for": "Used for monitoring TiKV ttl checker actions in the TTL category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_ttl_checker_actions{instance=~\"$instance\"}[30s])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6986,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TTL checker compact duration",
    "group": "TTL",
    "used_for": "Used for monitoring TiKV ttl checker compact duration in the TTL category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_ttl_checker_compact_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_ttl_checker_compact_duration_count{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7326,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TTL checker poll interval",
    "group": "TTL",
    "used_for": "Used for monitoring TiKV ttl checker poll interval in the TTL category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tikv_ttl_checker_poll_interval{instance=~\"$instance\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1069,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD requests",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd requests in the PD category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_pd_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1070,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD request duration (average)",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd request duration (average) in the PD category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_pd_request_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1215,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD heartbeats",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd heartbeats in the PD category. Helps track cluster health and communication between components.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_pd_heartbeat_message_total{instance=~\"$instance\"}[1m])) by (type)",
      "tikv_pd_pending_heartbeat_total{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1396,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD validate peers",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd validate peers in the PD category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_pd_validate_peer_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7985,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD reconnections",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd reconnections in the PD category. Helps track connection patterns and potential connection exhaustion.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "connection",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_pd_reconnect_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8376,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "PD forward status",
    "group": "PD",
    "used_for": "Used for monitoring TiKV pd forward status in the PD category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_pd_request_forwarded{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9963,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Pending TSO Requests",
    "group": "PD",
    "used_for": "Used for monitoring TiKV TSO (Timestamp Oracle) pending tso requests in the PD category. Helps track timestamp allocation performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_pd_pending_tso_request_total{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 10365,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store Slow Score",
    "group": "PD",
    "used_for": "Used for monitoring TiKV store slow score in the PD category. Helps track storage capacity and usage patterns.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_raftstore_slow_score{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 10366,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Inspected duration per server",
    "group": "PD",
    "used_for": "Used for monitoring TiKV inspected duration per server in the PD category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_inspect_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5993,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write IO bytes",
    "group": "IO Breakdown",
    "used_for": "Used for monitoring TiKV write io bytes in the IO Breakdown category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"write\"}[45s])) by (type)",
      "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"write\"}[45s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5994,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Read IO bytes",
    "group": "IO Breakdown",
    "used_for": "Used for monitoring TiKV read io bytes in the IO Breakdown category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"read\"}[45s])) by (type)",
      "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"read\"}[45s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5995,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "IO threshold",
    "group": "IO Breakdown",
    "used_for": "Used for monitoring TiKV io threshold in the IO Breakdown category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tikv_rate_limiter_max_bytes_per_sec{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7225,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rate Limiter Request Wait Duration",
    "group": "IO Breakdown",
    "used_for": "Used for monitoring TiKV rate limiter request wait duration in the IO Breakdown category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le, type))",
      "rate(tikv_rate_limiter_request_wait_duration_seconds_sum{instance=~\"$instance\"}[30s]) / rate(tikv_rate_limiter_request_wait_duration_seconds_count{instance=~\"$instance\"}[30s])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13132,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async write duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV storage async write duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[30s])) by (le))",
      "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[30s])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[30s]))",
      "sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13257,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13259,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Apply duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV apply duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_apply_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13261,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store propose wait duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store propose wait duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13263,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store batch wait duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store batch wait duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13265,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store send to write queue duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store send to write queue duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572857,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store send proposal duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store send proposal duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13269,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write kv db end duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store write kv db end duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13267,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store before write duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store before write duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13273,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store persist duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store persist duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13271,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write end duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store write end duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13277,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store commit but not persist duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store commit but not persist duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13275,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store commit and persist duration",
    "group": "Raft Waterfall",
    "used_for": "Used for monitoring TiKV store commit and persist duration in the Raft Waterfall category (P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
      "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[30s]))",
      "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[30s]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13279,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Process ready duration",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV process ready duration in the Raft IO category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\"$instance\", type='ready'}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13281,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Process ready duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% process ready duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\"$instance\", type='ready'}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13283,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write loop duration",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV store write loop duration in the Raft IO category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_store_write_loop_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13285,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Store write loop duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% store write loop duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_loop_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 39,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Append log duration",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV append log duration in the Raft IO category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13376,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Append log duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% append log duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_kvdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_send_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_callback_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3690,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Commit log duration",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV commit log duration in the Raft IO category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3688,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Commit log duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% commit log duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 31,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Apply log duration",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV apply log duration in the Raft IO category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 32,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Apply log duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% apply log duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13382,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store io task reschedule",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV store io task reschedule in the Raft IO category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)",
      "sum(tikv_raftstore_io_reschedule_pending_tasks_total{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13380,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Write task block duration per server",
    "group": "Raft IO",
    "used_for": "Used for monitoring TiKV 99% write task block duration per server in the Raft IO category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 108,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft proposals per ready",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV raft proposals per ready in the Raft Propose category (P99). Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft read/write proposals",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV raft read/write proposals in the Raft Propose category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"local_read|normal|read_index\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 119,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft read proposals per server",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV raft read proposals per server in the Raft Propose category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"local_read|read_index\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 120,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft write proposals per server",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV raft write proposals per server in the Raft Propose category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=\"normal\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 41,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Propose wait duration",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV propose wait duration in the Raft Propose category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 42,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Propose wait duration per server",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV 99% propose wait duration per server in the Raft Propose category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13524,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write wait duration",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV store write wait duration in the Raft Propose category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_store_write_task_wait_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13522,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Store write wait duration per server",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV 99% store write wait duration per server in the Raft Propose category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2535,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Apply wait duration",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV apply wait duration in the Raft Propose category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2536,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Apply wait duration per server",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV 99% apply wait duration per server in the Raft Propose category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572700,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write handle msg duration",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV store write handle msg duration in the Raft Propose category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_store_write_handle_msg_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572701,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Store write trigger size",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV store write trigger size in the Raft Propose category. Helps track storage capacity and usage patterns.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_store_write_trigger_wb_bytes_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1975,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft propose speed",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV raft propose speed in the Raft Propose category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(rate(tikv_raftstore_propose_log_size_sum{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1976,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Perf Context duration",
    "group": "Raft Propose",
    "used_for": "Used for monitoring TiKV perf context duration in the Raft Propose category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Ready handled",
    "group": "Raft Process",
    "used_for": "Used for monitoring TiKV ready handled in the Raft Process category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_ready_handled_total{instance=~\"$instance\"}[1m])) by (type)",
      "sum(rate(tikv_raftstore_raft_process_duration_secs_count{instance=~\"$instance\", type=\"ready\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 123,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Max duration of raft store events",
    "group": "Raft Process",
    "used_for": "Used for monitoring TiKV max duration of raft store events in the Raft Process category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1.0, sum(rate(tikv_raftstore_event_duration_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(1.0, sum(rate(tikv_broadcast_normal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7235,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Replica read lock checking duration",
    "group": "Raft Process",
    "used_for": "Used for monitoring TiKV replica read lock checking duration in the Raft Process category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_replica_read_lock_check_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572958,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Peer msg length distribution",
    "group": "Raft Process",
    "used_for": "Used for monitoring TiKV peer msg length distribution in the Raft Process category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_peer_msg_len_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1615,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Sent messages per server",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV sent messages per server in the Raft Message category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1616,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flush messages per server",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV flush messages per server in the Raft Message category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_server_raft_message_flush_total{instance=~\"$instance\"}[1m])) by (instance, reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 106,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Receive messages per server",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV receive messages per server in the Raft Message category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_server_raft_message_recv_total{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 11,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Messages",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV messages in the Raft Message category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 25,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Vote",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV vote in the Raft Message category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"vote\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1309,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft dropped messages",
    "group": "Raft Message",
    "used_for": "Used for monitoring TiKV raft dropped messages in the Raft Message category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_dropped_message_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 76,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Admin proposals",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV admin proposals in the Raft Admin category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 77,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Admin apply",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV admin apply in the Raft Admin category. Helps track Raft apply operations performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_admin_cmd_total{instance=~\"$instance\", status=\"success\", type!=\"compact\"}[1m]))  by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 70,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Check split",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV check split in the Raft Admin category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_check_split_total{instance=~\"$instance\", type!=\"ignore\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 71,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99.99% Check split duration",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV 99.99% check split duration in the Raft Admin category (P9999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3636,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Load base split event",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV load base split event in the Raft Admin category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(delta(tikv_load_base_split_event[1m])) by (type)",
      ""
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3637,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TopN QPS exceeds threshold",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV queries per second (QPS) in the Raft Admin category. Helps track workload volume and throughput changes.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_read_qps_topn{order=\"0\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572060,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Load base split duration",
    "group": "Raft Admin",
    "used_for": "Used for monitoring TiKV load base split duration in the Raft Admin category (P80/P90). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      80,
      90
    ],
    "variables": [
      "instance",
      "tidb_cluster"
    ],
    "promql": [
      "histogram_quantile(0.80, sum(rate(tikv_load_base_split_duration_seconds_bucket{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.90, sum(rate(tikv_load_base_split_duration_seconds_bucket{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (le, instance))",
      "sum(rate(tikv_load_base_split_duration_seconds_sum{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12882,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC write duration",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc write duration in the Raft Log category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12886,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC kv sync duration",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc kv sync duration in the Raft Log category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12881,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC write operations ",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc write operations  in the Raft Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12972,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC deleted keys",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc deleted keys in the Raft Log category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_sum{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12884,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC seek operations ",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc seek operations  in the Raft Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_log_gc_seek_operations_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12973,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC write batch size",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc write batch size in the Raft Log category (P95/P99). Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12883,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC purge duration",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc purge duration in the Raft Log category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12887,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log lag",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log lag in the Raft Log category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_log_lag_sum{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12975,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log gc skipped",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc skipped in the Raft Log category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_log_gc_skipped{instance=~\"$instance\"}[1m])) by (instance, reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12974,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log GC failed",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log gc failed in the Raft Log category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_raft_log_gc_failed{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572229,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log fetch",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log fetch in the Raft Log category. Helps track Raft consensus performance and replication health.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_entry_fetches{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572555,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Raft log async fetch task duration",
    "group": "Raft Log",
    "used_for": "Used for monitoring TiKV raft log async fetch task duration in the Raft Log category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance",
      "tidb_cluster"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "sum(tikv_worker_pending_task_total{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\", name=~\"raftlog-fetch-worker\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2292,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Local reader requests",
    "group": "Local reader",
    "used_for": "Used for monitoring TiKV local reader requests in the Local reader category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_local_read_reject_total{instance=~\"$instance\"}[1m])) by (instance, reason)",
      "sum(rate(tikv_raftstore_local_read_executed_requests{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{instance=~\"$instance\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4194,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Time used by level",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV time used by level in the Unified Read Pool category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_multilevel_level_elapsed{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (level)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4196,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Level 0 chance",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV level 0 chance in the Unified Read Pool category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_multilevel_level0_chance{instance=~\"$instance\", name=\"unified-read-pool\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4198,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Running tasks",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV running tasks in the Unified Read Pool category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(avg_over_time(tikv_unified_read_pool_running_tasks{}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572469,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Unified Read Pool Wait Duration",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV unified read pool wait duration in the Unified Read Pool category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\", name=~\"unified-read.*\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4199,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Duration of One Time Slice",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV duration of one time slice in the Unified Read Pool category (P50/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      50,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.50, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4202,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Task Execute Duration",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV task execute duration in the Unified Read Pool category (P50/P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      50,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.50, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4204,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Task Schedule Times",
    "group": "Unified Read Pool",
    "used_for": "Used for monitoring TiKV task schedule times in the Unified Read Pool category (P50/P95/P99/P999). Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [
      50,
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.50, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage command total",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage command total in the Storage category. Helps track storage capacity and usage patterns.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_command_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async request error",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage async request error in the Storage category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_engine_async_request_total{instance=~\"$instance\", status!~\"all|success\"}[1m])) by (status)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async snapshot duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage async snapshot duration in the Storage category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 109,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async write duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage async write duration in the Storage category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 20000,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async snapshot duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage async snapshot duration in the Storage category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))",
      "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"snapshot\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"snapshot\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 20001,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Storage async write duration",
    "group": "Storage",
    "used_for": "Used for monitoring TiKV storage async write duration in the Storage category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le))",
      "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763574798,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Thread CPU Usage",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV CPU utilization in the Resource Control category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raft_io.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"status_server.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"pd_worker_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"grpc_server_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"unified_read_pool_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"remote_coproces.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573967,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request wait count",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV request wait count in the Resource Control category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_request_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance,type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573801,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Keyspace read MBps",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV keyspace read mbps in the Resource Control category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_active_keyspace_read_bytes{instance=~\"$instance\"}[1m])) by (instance,keyspace_id)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763574133,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Average request wait duration",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV average request wait duration in the Resource Control category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_request_wait_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (instance,type) / sum(rate(tikv_request_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance,type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763574299,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% request wait duration",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV 99% request wait duration in the Resource Control category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763574465,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "9999% request wait duration",
    "group": "Resource Control",
    "used_for": "Used for monitoring TiKV 9999% request wait duration in the Resource Control category (P9999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tikv_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9552,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler flow",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV scheduler flow in the Flow Control category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_scheduler_write_flow{instance=~\"$instance\"}) by (instance)",
      "sum(tikv_scheduler_throttle_flow{instance=~\"$instance\"}) by (instance) != 0"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9553,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler discard ratio",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV scheduler discard ratio in the Flow Control category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_scheduler_discard_ratio{instance=~\"$instance\"}) by (instance) / 10000000"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 11512,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Throttle duration",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV throttle duration in the Flow Control category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_scheduler_throttle_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9946,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction pending bytes",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV compaction pending bytes in the Flow Control category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"kv\"}) by (cf)",
      "max(tikv_scheduler_pending_compaction_bytes{instance=~\"$instance\"}) by (instance) / 10000000"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9947,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flow controller actions",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV flow controller actions in the Flow Control category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_throttle_action_total{instance=~\"$instance\"}[1m])) by (cf, type)",
      "sum(rate(kv_engine_throttle_action_total{instance=~\"$instance\"}[1m])) by (limiter, action)",
      "sum(rate(kv_engine_limiter_throttle_counter{instance=~\"$instance\"}[1m])) by (limiter)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9944,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flow controller factors",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV flow controller factors in the Flow Control category. Helps detect when flow control is throttling operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tikv_scheduler_l0{instance=~\"$instance\"}) by (instance)",
      "max(tikv_scheduler_memtable{instance=~\"$instance\"}) by (instance)",
      "max(tikv_scheduler_l0_avg{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572363,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Txn command throttled duration",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV txn command throttled duration in the Flow Control category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_txn_command_throttle_time_total{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572365,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Non-txn command throttled duration",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV non-txn command throttled duration in the Flow Control category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_non_txn_command_throttle_time_total{instance=~\"$instance\"}[1m])) by (type) "
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 11906,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler throttled CF",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV scheduler throttled cf in the Flow Control category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_scheduler_throttle_cf{instance=~\"$instance\"} != 0"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 10338,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flush/L0 flow",
    "group": "Flow Control",
    "used_for": "Used for monitoring TiKV flush/l0 flow in the Flow Control category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_scheduler_l0_flow{instance=~\"$instance\"}) by (instance, cf)",
      "sum(tikv_scheduler_flush_flow{instance=~\"$instance\"}) by (instance, cf)",
      "sum(tikv_scheduler_l0_flow{instance=~\"$instance\"}) by (instance)",
      "sum(tikv_scheduler_flush_flow{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 168,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler stage total",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler stage total in the Scheduler - $command category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\", type=\"$command\"}[1m]))",
      "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\", type=\"$command\"}[1m])) by (stage)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler command duration",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler command duration in the Scheduler - $command category (P95/P99/P9999/P9999999999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      9999,
      9999999999
    ],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "sum(rate(tikv_scheduler_command_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m])) ",
      "histogram_quantile(0.9999999999, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.9999, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 194,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler latch wait duration",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler latch wait duration in the Scheduler - $command category (P95/P99/P9999/P99999999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      9999,
      99999999
    ],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m])) ",
      "histogram_quantile(0.9999, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.99999999, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 195,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler keys read",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler keys read in the Scheduler - $command category (P95/P99). Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "sum(rate(tikv_scheduler_kv_command_key_read_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{instance=~\"$instance\", type=\"$command\"}[1m])) "
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 373,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler keys written",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler keys written in the Scheduler - $command category (P95/P99). Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "sum(rate(tikv_scheduler_kv_command_key_write_sum{instance=~\"$instance\", type=\"$command\"}[1m])) by (type) / sum(rate(tikv_scheduler_kv_command_key_write_count{instance=~\"$instance\", type=\"$command\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 560,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler scan details",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler scan details in the Scheduler - $command category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", req=\"$command\"}[1m])) by (tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 675,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler scan details [lock]",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler scan details [lock] in the Scheduler - $command category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"lock\", req=\"$command\"}[1m])) by (tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 829,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler scan details [write]",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler scan details [write] in the Scheduler - $command category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"write\", req=\"$command\"}[1m])) by (tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 830,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler scan details [default]",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler scan details [default] in the Scheduler - $command category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"default\"}[1m])) by (tag,req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572710,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler command read duration",
    "group": "Scheduler - $command",
    "used_for": "Used for monitoring TiKV scheduler command read duration in the Scheduler - $command category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "command",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))",
      "sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) by (type) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573468,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler command count",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler command count in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573124,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler command duration",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler command duration in the Scheduler category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\",}[1m])) by (le, type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 167,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler stage total",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler stage total in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\"}[1m])) by (stage)",
      "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\"}[1m])) by (stage)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3834,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler writing bytes",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler writing bytes in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_scheduler_writing_bytes{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler priority commands",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler priority commands in the Scheduler category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_scheduler_commands_pri_total{instance=~\"$instance\"}[1m])) by (priority)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 193,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scheduler pending commands",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV scheduler pending commands in the Scheduler category. Helps track scheduling operations and potential bottlenecks.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_scheduler_contex_total{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572468,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Txn Scheduler Pool Wait Duration",
    "group": "Scheduler",
    "used_for": "Used for monitoring TiKV txn scheduler pool wait duration in the Scheduler category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "transaction",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\", name=~\"sched-worker.*\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 121,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC tasks",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc tasks in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)",
      "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)",
      "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)",
      "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2224,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC tasks duration",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc tasks duration in the GC category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))",
      "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))",
      "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))",
      "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 969,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TiDB GC seconds",
    "group": "GC",
    "used_for": "Used for monitoring TiDB tidb gc seconds in the GC category. Helps track garbage collection operations.",
    "component": "TIDB",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket{}[1m])) by (instance, le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 966,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TiDB GC worker actions",
    "group": "GC",
    "used_for": "Used for monitoring TiDB tidb gc worker actions in the GC category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIDB",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(tidb_tikvclient_gc_worker_actions_total{}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2823,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "ResolveLocks Progress",
    "group": "GC",
    "used_for": "Used for monitoring TiKV resolvelocks progress in the GC category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(tidb_tikvclient_range_task_stats{type=~\"resolve-locks.*\"}) by (result)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2821,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TiKV Auto GC Progress",
    "group": "GC",
    "used_for": "Used for monitoring TiKV tikv auto gc progress in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_gcworker_autogc_processed_regions{instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2589,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC speed",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc speed in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) by (key_mode)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2822,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TiKV Auto GC SafePoint",
    "group": "GC",
    "used_for": "Used for monitoring TiKV tikv auto gc safepoint in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(tikv_gcworker_autogc_safe_point{}) by (instance) / (2^18)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 27,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC lifetime",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc lifetime in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 28,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC interval",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc interval in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6596,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC in Compaction Filter",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc in compaction filter in the GC category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_gc_compaction_filtered{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_filter_skip{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_mvcc_rollback{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_filter_orphan_versions{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_filter_perform{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_failure{instance=~\"$instance\"}[1m])) by (key_mode,type)",
      "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_met{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_handled{instance=~\"$instance\"}[1m])) by (key_mode)",
      "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_wasted{instance=~\"$instance\"}[1m])) by (key_mode)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8767,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC scan write details",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc scan write details in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (key_mode,tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8768,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "GC scan default details",
    "group": "GC",
    "used_for": "Used for monitoring TiKV gc scan default details in the GC category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"default\"}[1m])) by (key_mode,tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 35,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rate snapshot message",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV rate snapshot message in the Snapshot category. Helps track snapshot operations and their impact on performance.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 37,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% Handle snapshot duration",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV 99% handle snapshot duration in the Snapshot category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 38,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Snapshot state count",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV snapshot state count in the Snapshot category. Helps track snapshot operations and their impact on performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 44,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99.99% Snapshot size",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV 99.99% snapshot size in the Snapshot category (P9999). Helps track snapshot operations and their impact on performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 43,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99.99% Snapshot KV count",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV 99.99% snapshot kv count in the Snapshot category (P9999). Helps track snapshot operations and their impact on performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 36,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Snapshot Actions",
    "group": "Snapshot",
    "used_for": "Used for monitoring TiKV snapshot actions in the Snapshot category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_raftstore_snapshot_total{instance=~\"$instance\"}[1m])) by (type, status)",
      "sum(delta(tikv_raftstore_clean_region_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 59,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Worker handled tasks",
    "group": "Task",
    "used_for": "Used for monitoring TiKV worker handled tasks in the Task category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1395,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Worker pending tasks",
    "group": "Task",
    "used_for": "Used for monitoring TiKV worker pending tasks in the Task category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_worker_pending_task_total{instance=~\"$instance\"}) by (name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1876,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "FuturePool handled tasks",
    "group": "Task",
    "used_for": "Used for monitoring TiKV futurepool handled tasks in the Task category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1877,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "FuturePool pending tasks",
    "group": "Task",
    "used_for": "Used for monitoring TiKV futurepool pending tasks in the Task category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3062,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request duration",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV request duration in the Coprocessor Overview category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 16,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request duration",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV request duration in the Coprocessor Overview category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))",
      "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 74,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Requests",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV total requests in the Coprocessor Overview category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3128,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Request Errors",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV total request errors in the Coprocessor Overview category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 52,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total KV Cursor Operations",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV total kv cursor operations in the Coprocessor Overview category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3129,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "KV Cursor Operations",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV kv cursor operations in the Coprocessor Overview category (P99). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req))  ",
      "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req))  "
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2118,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total RocksDB Perf Statistics",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV total rocksdb perf statistics in the Coprocessor Overview category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 551,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Response Size",
    "group": "Coprocessor Overview",
    "used_for": "Used for monitoring TiKV total response size in the Coprocessor Overview category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 113,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Handle duration",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV handle duration in the Coprocessor Detail category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))",
      "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 117,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "95% Handle duration by store",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV 95% handle duration by store in the Coprocessor Detail category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      95
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 111,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Wait duration",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV wait duration in the Coprocessor Detail category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))",
      "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 116,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "95% Wait duration by store",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV 95% wait duration by store in the Coprocessor Detail category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "store",
    "percentile": [
      95
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3195,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total DAG Requests",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total dag requests in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3264,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total DAG Executors",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total dag executors in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 552,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Ops Details (Table Scan)",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total ops details (table scan) in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3263,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Ops Details (Index Scan)",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total ops details (index scan) in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 122,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Ops Details by CF (Table Scan)",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total ops details by cf (table scan) in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 554,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Ops Details by CF (Index Scan)",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV total ops details by cf (index scan) in the Coprocessor Detail category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 7594,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memory lock checking duration",
    "group": "Coprocessor Detail",
    "used_for": "Used for monitoring TiKV memory usage in the Coprocessor Detail category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_mem_lock_check_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2108,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Threads state",
    "group": "Threads",
    "used_for": "Used for monitoring TiKV threads state in the Threads category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)",
      "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2258,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Threads IO",
    "group": "Threads",
    "used_for": "Used for monitoring TiKV threads io in the Threads category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "topk(20, sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2660,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Thread Voluntary Context Switches",
    "group": "Threads",
    "used_for": "Used for monitoring TiKV thread voluntary context switches in the Threads category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "topk(20, max(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (name) > 200)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2661,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Thread Nonvoluntary Context Switches",
    "group": "Threads",
    "used_for": "Used for monitoring TiKV thread nonvoluntary context switches in the Threads category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "topk(20, max(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (name) > 100)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 138,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Get operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV get operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))",
      "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))",
      "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))",
      "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 82,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Get duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV get duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})",
      "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})",
      "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})",
      "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 129,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Seek operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV seek operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))",
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))",
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))",
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))",
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))",
      "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 125,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Seek duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV seek duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})",
      "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})",
      "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})",
      "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 139,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))",
      "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))",
      "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 126,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})",
      "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})",
      "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})",
      "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 137,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "WAL sync operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV wal sync operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 130,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write WAL duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write wal duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})",
      "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})",
      "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})",
      "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 128,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 135,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "WAL sync duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV wal sync duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})",
      "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})",
      "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})",
      "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2453,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction guard actions",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction guard actions in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_raftstore_compaction_guard_action_total{instance=~\"$instance\", cf=~\"default|write\"}[1m])) by (cf, type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 136,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})",
      "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})",
      "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})",
      "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 140,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "SST read duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV sst read duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})",
      "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})",
      "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})",
      "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2451,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction reason",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction reason in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 102,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Block cache size",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV block cache size in the RocksDB - $db category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "topk(20, avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 88,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memtable hit",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV memtable hit in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 467,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Block cache flow",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV block cache flow in the RocksDB - $db category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 80,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Block cache hit",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV block cache hit in the RocksDB - $db category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[1m])))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[1m])))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[1m])))",
      "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 132,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Keys flow",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV keys flow in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))",
      "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 468,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Block cache operations",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV block cache operations in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))",
      "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 85,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Read flow",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV read flow in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 131,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total keys",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV total keys in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 86,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write flow",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write flow in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 133,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Bytes / Read",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV bytes / read in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})",
      "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})",
      "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})",
      "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 90,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction flow",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction flow in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))",
      "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))",
      "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 134,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Bytes / Write",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV bytes / write in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})",
      "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})",
      "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})",
      "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 518,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Read amplication",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV read amplication in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 127,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compaction pending bytes",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compaction pending bytes in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}) by (cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 516,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Number of snapshots",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV number of snapshots in the RocksDB - $db category. Helps track snapshot operations and their impact on performance.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 863,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Compression ratio",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV compression ratio in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (cf, level)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2002,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Number files at each level",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV number files at each level in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 517,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Oldest snapshots duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV oldest snapshots duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2381,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Stall conditions changed of each CF",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV stall conditions changed of each cf in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2003,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Ingest SST duration seconds",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV ingest sst duration seconds in the RocksDB - $db category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le))",
      "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\", db=\"$db\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\", db=\"$db\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2452,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write Stall Reason",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write stall reason in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_engine_write_stall_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 87,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write stall duration",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV write stall duration in the RocksDB - $db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "max(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})",
      "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})",
      "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})",
      "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12712,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Ingestion picked level",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV ingestion picked level in the RocksDB - $db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_engine_ingestion_picked_level_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 103,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memtable size",
    "group": "RocksDB - $db",
    "used_for": "Used for monitoring TiKV memtable size in the RocksDB - $db category.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "db",
      "instance"
    ],
    "promql": [
      "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12893,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Persist Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine persist duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_persist_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_persist_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_persist_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_persist_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573290,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Rotate Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine rotate duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_rotate_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_rotate_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_rotate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_rotate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573292,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine WAL Write Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine wal write duration in the Raft Engine category (P95/P99/P999/P99999999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999,
      99999999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_wal_write_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_wal_write_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99999999, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573293,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Apply Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine apply duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_apply_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_apply_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573294,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Truncate Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine truncate duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_truncate_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_truncate_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_truncate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_truncate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573295,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Fetch Entries Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine fetch entries duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_fetch_entries_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_fetch_entries_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_fetch_entries_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_fetch_entries_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573296,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Compact WAL Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine compact wal duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_compact_wal_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_compact_wal_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_compact_wal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_compact_wal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573297,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Task Snapshot Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine task snapshot duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_take_snapshot_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_take_snapshot_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_take_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_take_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573299,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Region Write Batch Size",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine region write batch size in the Raft Engine category (P99/P999). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_region_write_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_region_write_batch_size_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(raft_engine_region_write_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_region_write_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573300,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Log File GC Size",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine log file gc size in the Raft Engine category (P99/P999). Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_log_file_gc_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_log_file_gc_size_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(raft_engine_log_file_gc_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_log_file_gc_size_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573298,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Backup Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine backup duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_backup_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_backup_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_backup_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_backup_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573301,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine Backup Counter",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine backup counter in the Raft Engine category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_backup_counter{instance=~\"$instance\"}[1m])) by (status)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573302,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Rfengine DFS Worker Unhealthy",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV rfengine dfs worker unhealthy in the Raft Engine category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "1 - raft_engine_dfs_worker_healthy"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763573291,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write Duration",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV write duration in the Raft Engine category (P95/P99/P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_write_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_write_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.95, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12892,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Operation",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV operation in the Raft Engine category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_write_apply_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "sum(rate(raft_engine_read_entry_duration_seconds_count{instance=~\"$instance\"}[1m]))",
      "sum(rate(raft_engine_read_message_duration_seconds_count{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12895,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write Duration Breakdown (99%)",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV write duration breakdown (99%) in the Raft Engine category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_write_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_write_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12896,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flow",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV flow in the Raft Engine category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_write_size_sum{instance=~\"$instance\"}[1m]))",
      "sum(rate(raft_engine_background_rewrite_bytes_sum{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12933,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "WAL Duration Breakdown (999%)",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV wal duration breakdown (999%) in the Raft Engine category (P999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.999, sum(rate(raft_engine_write_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_sync_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_allocate_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_rotate_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12898,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Bytes / Written",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV bytes / written in the Raft Engine category (P99/P999).",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99,
      999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(raft_engine_write_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_write_size_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(raft_engine_write_size_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.999, sum(rate(raft_engine_write_size_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12897,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Other Durations (99%)",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV other durations (99%) in the Raft Engine category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(raft_engine_read_entry_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_read_message_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.99, sum(rate(raft_engine_purge_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12899,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "File Count",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV file count in the Raft Engine category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(raft_engine_log_file_count{instance=~\"$instance\"}) by (type)",
      "avg(raft_engine_swap_file_count{instance=~\"$instance\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12934,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Entry Count",
    "group": "Raft Engine",
    "used_for": "Used for monitoring TiKV entry count in the Raft Engine category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(raft_engine_log_entry_count{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3555,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file count",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file count in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$titan_db\"})",
      "sum(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$titan_db\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3557,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file size in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$titan_db\"})",
      "sum(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$titan_db\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3523,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Live blob size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV live blob size in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$titan_db\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4655,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob cache size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob cache size in the Titan - $titan_db category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "topk(20, avg(tikv_engine_blob_cache_size_bytes{instance=~\"$instance\", db=\"$titan_db\"}) by(cf, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4020,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob cache hit",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob cache hit in the Titan - $titan_db category. Helps evaluate cache effectiveness and hit rates.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_cache_efficiency{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_cache_hit\"}[1m])) / (sum(rate(tikv_engine_blob_cache_efficiency{db=\"$titan_db\", type=\"blob_cache_hit\"}[1m])) + sum(rate(tikv_engine_blob_cache_efficiency{db=\"$titan_db\", type=\"blob_cache_miss\"}[1m])))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4023,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Iter touched blob file count",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV iter touched blob file count in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_average\"})",
      "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile95\"})",
      "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile99\"})",
      "max(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4025,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file discardable ratio distribution",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file discardable ratio distribution in the Titan - $titan_db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(tikv_engine_titandb_blob_file_discardable_ratio{instance=~\"$instance\", db=\"$titan_db\"}) by (ratio)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3414,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob key size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob key size in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_average\"})",
      "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile95\"})",
      "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile99\"})",
      "max(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3446,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob value size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob value size in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_average\"})",
      "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile95\"})",
      "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile99\"})",
      "max(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3746,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob get operations",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob get operations in the Titan - $titan_db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_get\"}[2m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3655,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob get duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob get duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)",
      "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)",
      "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)",
      "max(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3338,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob iter operations",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob iter operations in the Titan - $titan_db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_seek\"}[2m]))",
      "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_prev\"}[2m]))",
      "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_next\"}[2m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3412,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob seek duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob seek duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)",
      "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)",
      "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)",
      "max(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4092,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob next duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob next duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)",
      "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)",
      "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)",
      "max(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4093,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob prev duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob prev duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)",
      "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)",
      "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)",
      "max(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3645,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob keys flow",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob keys flow in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[30s])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3643,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob bytes flow",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob bytes flow in the Titan - $titan_db category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3657,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file read duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file read duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_average\"})",
      "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile99\"})",
      "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile95\"})",
      "max(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3408,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file write duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file write duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_average\"})",
      "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile99\"})",
      "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile95\"})",
      "max(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3651,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file sync operations",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file sync operations in the Titan - $titan_db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_file_synced{instance=~\"$instance\", db=\"$titan_db\"}[2m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3653,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob file sync duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob file sync duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_average\"})",
      "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile95\"})",
      "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile99\"})",
      "max(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5018,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC action",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc action in the Titan - $titan_db category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_gc_action_count{instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3410,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC duration",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc duration in the Titan - $titan_db category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_average\"})",
      "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile95\"})",
      "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile99\"})",
      "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3649,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC keys flow",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc keys flow in the Titan - $titan_db category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[30s])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3340,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC bytes flow",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc bytes flow in the Titan - $titan_db category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[30s])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4021,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC input file size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc input file size in the Titan - $titan_db category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_average\"})",
      "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile95\"})",
      "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile99\"})",
      "max(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4022,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC output file size",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc output file size in the Titan - $titan_db category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_average\"})",
      "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile95\"})",
      "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile99\"})",
      "max(tikv_engine_blob_gc_outputt_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_max\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3344,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Blob GC file count",
    "group": "Titan - $titan_db",
    "used_for": "Used for monitoring TiKV blob gc file count in the Titan - $titan_db category. Helps track garbage collection operations.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance",
      "titan_db"
    ],
    "promql": [
      "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2991,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Lock Manager Thread CPU",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV CPU utilization in the Pessimistic Locking category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"waiter_manager.*\"}[1m])) by (instance, name)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"deadlock_detect.*\"}[1m])) by (instance, name)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2877,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Lock Manager Handled tasks",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV lock manager handled tasks in the Pessimistic Locking category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_lock_manager_task_counter{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2993,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Waiter lifetime duration",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV waiter lifetime duration in the Pessimistic Locking category (P99/P9999). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99,
      9999
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\"$instance\"}[1m])) by (le))",
      "histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4018,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Lock Waiting Queue",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV lock waiting queue in the Pessimistic Locking category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(max_over_time(tikv_lock_manager_wait_table_status{instance=~\"$instance\"}[15s])) by (type)",
      "sum(max_over_time(tikv_lock_wait_queue_entries_gauge_vec{instance=~\"$instance\"}[15s])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2995,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Deadlock detect duration",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV deadlock detect duration in the Pessimistic Locking category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_lock_manager_detect_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_lock_manager_detect_duration_count{instance=~\"$instance\"}[1m]))",
      "histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{instance=~\"$instance\"}[1m])) by (le))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2934,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Detect error",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV detect error in the Pessimistic Locking category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_lock_manager_error_counter{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763571993,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "cloud request",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV cloud request in the Pessimistic Locking category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_cloud_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (cloud, req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572093,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total pessimistic locks memory size",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV memory usage in the Pessimistic Locking category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_pessimistic_lock_memory_size{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4019,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Deadlock detector leader",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV deadlock detector leader in the Pessimistic Locking category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{instance=~\"$instance\"}[15s])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572094,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "In-memory pessimistic locking result",
    "group": "Pessimistic Locking",
    "used_for": "Used for monitoring TiKV memory usage in the Pessimistic Locking category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_in_memory_pessimistic_locking{instance=~\"$instance\"}[1m])) by (result)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8385,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Resolved TS Worker CPU",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV CPU utilization in the Resolved-TS category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"resolved_ts.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9162,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Advance ts Worker CPU",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV CPU utilization in the Resolved-TS category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"advance_ts.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9164,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Scan lock Worker CPU",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV CPU utilization in the Resolved-TS category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"inc_scan.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8387,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Max Resolved TS gap",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV max resolved ts gap in the Resolved-TS category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_resolved_ts_min_resolved_ts_gap_millis{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572077,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Max Leader Resolved TS gap",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV max leader resolved ts gap in the Resolved-TS category. Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_resolved_ts_min_leader_resolved_ts_gap_millis{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572078,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Min Resolved TS Region",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV min resolved ts region in the Resolved-TS category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance",
      "tidb_cluster"
    ],
    "promql": [
      "sum(tikv_resolved_ts_min_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572079,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Min Leader Resolved TS Region",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV min leader resolved ts region in the Resolved-TS category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance",
      "tidb_cluster"
    ],
    "promql": [
      "sum(tikv_resolved_ts_min_leader_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9168,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Check leader duration",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV check leader duration in the Resolved-TS category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_resolved_ts_check_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8377,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Observe region status",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV observe region status in the Resolved-TS category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_resolved_ts_region_resolve_status{instance=~\"$instance\"}) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12308,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% CheckLeader request region count",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV 99% checkleader request region count in the Resolved-TS category (P99). Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 9166,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Fail advance ts count",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV fail advance ts count in the Resolved-TS category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(delta(tikv_resolved_ts_fail_advance_count{instance=~\"$instance\"}[1m])) by (instance, reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8379,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Lock heap size",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV lock heap size in the Resolved-TS category. Helps identify lock contention and pessimistic locking overhead.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tikv_resolved_ts_lock_heap_bytes{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8383,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "99% CheckLeader request size",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV 99% checkleader request size in the Resolved-TS category (P99). Helps track leader distribution and election status.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_size_bytes_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
      "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{instance=~\"$instance\"}[1m])) by (le, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 8381,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Pending command size",
    "group": "Resolved-TS",
    "used_for": "Used for monitoring TiKV pending command size in the Resolved-TS category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "avg(tikv_resolved_ts_channel_penging_cmd_bytes_total{instance=~\"$instance\"}) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 2696,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Allocator Stats",
    "group": "Memory",
    "used_for": "Used for monitoring TiKV allocator stats in the Memory category.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_allocator_stats{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3924,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup CPU Utilization",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV CPU utilization in the Backup & Import category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"b.*k.*w.*k.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_io\"}[1m])) by (instance)",
      "tikv_backup_softlimit{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3926,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Thread Count",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup thread count in the Backup & Import category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_backup_thread_pool_size{instance=~\"$instance\"}) by(instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5264,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Errors",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup errors in the Backup & Import category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "delta(tikv_backup_error_counter{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3927,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Write CF SST Size",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup write cf sst size in the Backup & Import category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_backup_range_size_bytes_bucket{instance=~\"$instance\", cf=\"write\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5266,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Default CF SST Size",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup default cf sst size in the Backup & Import category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_backup_range_size_bytes_bucket{instance=~\"$instance\", cf=\"default\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3929,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup SST Generation Throughput",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup sst generation throughput in the Backup & Import category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_backup_range_size_bytes_sum{instance=~\"$instance\"}[1m]))",
      "rate(tikv_backup_range_size_bytes_sum{}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5597,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Scan SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup scan sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3931,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Scan SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup scan sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=\"scan\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6905,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup Save SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup save sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=~\"save.*\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3928,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Backup SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV backup sst duration in the Backup & Import category (P95/P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95,
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))",
      "sum(rate(tikv_backup_range_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 3930,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "External Storage Create Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV external storage create duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4936,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "External Storage Create Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV external storage create duration in the Backup & Import category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,type))",
      "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,type))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5267,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Checksum Request Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV checksum request duration in the Backup & Import category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "request",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[1m])) by (le,req))",
      "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[1m])) by (le,req))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5269,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "IO Utilization",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV io utilization in the Backup & Import category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(node_disk_io_time_seconds_total{}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5925,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import CPU Utilization",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV CPU utilization in the Backup & Import category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)",
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0",
      "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5926,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Thread Count",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import thread count in the Backup & Import category. Helps identify thread pool saturation and concurrency bottlenecks.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5932,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Errors",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import errors in the Backup & Import category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "delta(tikv_import_error_counter{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5931,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import RPC Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import rpc duration in the Backup & Import category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\"}[1m])) by (le, request))",
      "histogram_quantile(0.5, sum(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\"}[1m])) by (le, request, instance))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6267,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import RPC Ops",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import rpc ops in the Backup & Import category.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_import_rpc_duration_count{instance=~\"$instance\", request!=\"switch_mode\"}[30s])) by (instance, request)",
      "sum(rate(tikv_import_rpc_duration_count{instance=~\"$instance\", request!=\"switch_mode\"}[30s])) by (request)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5930,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Write/Download RPC Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import write/download rpc duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\", request=~\"download|write\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5929,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Wait Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import wait duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"queue\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 6906,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Read SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import read sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"read\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5928,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Rewrite SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import rewrite sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"rewrite\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5939,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Ingest RPC Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import ingest rpc duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\", request=~\"ingest\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5938,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Ingest SST Duration",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import ingest sst duration in the Backup & Import category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_ingest_duration_bucket{instance=~\"$instance\", type=~\"ingest\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5937,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Ingest SST Bytes",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import ingest sst bytes in the Backup & Import category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(rate(tikv_import_ingest_byte{instance=~\"$instance\"}[1m])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 5927,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Download SST Throughput",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import download sst throughput in the Backup & Import category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_import_download_bytes_sum{instance=~\"$instance\"}[1m])) by (instance)",
      "sum(rate(tikv_import_download_bytes_sum{instance=~\"$instance\"}[1m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12309,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Local Write keys",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import local write keys in the Backup & Import category.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "delta(tikv_import_local_write_keys{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12310,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Import Local Write bytes",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV import local write bytes in the Backup & Import category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "delta(tikv_import_local_write_bytes{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572861,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "TTL Expired",
    "group": "Backup & Import",
    "used_for": "Used for monitoring TiKV ttl expired in the Backup & Import category.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(tikv_backup_raw_expired_count{instance=~\"$instance\"}) by (instance)",
      "sum(tikv_backup_raw_expired_count{instance=~\"$instance\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4464,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Encryption data keys",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV encryption data keys in the Encryption category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_encryption_data_key_storage_total{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4554,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Encrypted files",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV encrypted files in the Encryption category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_encryption_file_num{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4555,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Encryption initialized",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV encryption initialized in the Encryption category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_encryption_is_initialized{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4556,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Encryption meta files size",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV encryption meta files size in the Encryption category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tikv_encryption_meta_file_size_bytes{}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4557,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Encrypt/decrypt data nanos",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV encrypt/decrypt data nanos in the Encryption category.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"encrypt_data_nanos\"}[1m])) by (req)",
      "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"decrypt_data_nanos\"}[1m])) by (req)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 4559,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Read/write encryption meta duration",
    "group": "Encryption",
    "used_for": "Used for monitoring TiKV read/write encryption meta duration in the Encryption category (P95). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      95
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "histogram_quantile(1, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, operation))",
      "histogram_quantile(0.95, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, operation))",
      "sum(rate(tikv_encryption_write_read_file_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (le, type, operation) / sum(rate(tikv_encryption_write_read_file_duration_seconds_count{instance=~\"$instance\"}[1m])) by (le, type, operation)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14361,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Endpoint Status",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV endpoint status in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_log_backup_enabled{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14507,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Average Flush Size ",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV average flush size  in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]) / on(instance) increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14363,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flushed Files (Last 30m) Per Host",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV flushed files (last 30m) per host in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "round(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14508,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flush Times (Last 30m)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV flush times (last 30m) in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "round(sum(increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14362,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Total Flushed Size (Last 30m)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV total flushed size (last 30m) in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14907,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Task Status",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV task status in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "min(tikv_log_backup_task_status{instance=~\"$instance\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15361,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Advancer Owner",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV advancer owner in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "tidb_log_backup_advancer_owner > 0"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14911,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flush Files (Last 30m)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV flush files (last 30m) in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "round(sum(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m])))"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13262,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "CPU Usage",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV CPU utilization in the Backup Log category. Helps identify CPU bottlenecks and resource pressure.",
    "component": "TIKV",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}[2m])) by (instance)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12843,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Handle Event Rate",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV handle event rate in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "rate(tikv_log_backup_handle_kv_batch_sum{instance=~\"$instance\"}[1m])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14135,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Initial Scan Generate Event Throughput",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV initial scan generate event throughput in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "rate(tikv_log_backup_incremental_scan_bytes_sum{instance=~\"$instance\"}[$__rate_interval])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14774,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Abnormal Checkpoint TS Lag",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV abnormal checkpoint ts lag in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "time() * 1000 - max(tidb_log_backup_last_checkpoint / 262144 > 0) by (task)",
      "time() * 1000"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13100,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Memory Of Events",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV memory usage in the Backup Log category. Helps identify memory pressure and potential OOM risks.",
    "component": "TIKV",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_log_backup_heap_memory{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14630,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Observed Region Count",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV observed region count in the Backup Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "tikv_log_backup_observed_region{instance=~\"$instance\"}",
      "sum(tikv_log_backup_observed_region{instance=~\"$instance\"})"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13101,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Errors",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV errors in the Backup Log category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "increase(tikv_log_backup_errors{instance=~\"$instance\"}[$__interval])",
      "tikv_log_backup_errors{instance=~\"$instance\"}"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14910,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Checkpoint TS of Tasks",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV checkpoint ts of tasks in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "max(tidb_log_backup_last_checkpoint{instance=~\"$instance\"} / 262144 > 0) by (task)",
      "time() * 1000"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14908,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Fatal Errors",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV fatal errors in the Backup Log category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "increase(tikv_log_backup_fatal_errors{instance=~\"$instance\"}[$__interval])",
      ""
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14078,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Flush Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV flush duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_flush_duration_sec_bucket{instance=~\"$instance\", stage=~\"save_files\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14136,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Initial scanning duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV initial scanning duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_initial_scan_duration_sec_bucket{instance=~\"$instance\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13934,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Convert Raft Event duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV convert raft event duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"to_stream_event\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12840,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Wait for Lock Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV wait for lock duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"get_router_lock\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15059,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Command Batch Size",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV command batch size in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_handle_kv_batch_bucket{instance=~\"$instance\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 12841,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Save to Temp File Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV save to temp file duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"save_to_temp_file\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13552,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Write to Temp File Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV write to temp file duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{instance=~\"$instance\", stage=\"write_to_tempfile\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 13551,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "System Write Call Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV system write call duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{instance=~\"$instance\", stage=\"syscall_write\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14914,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Internal Message Type",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV internal message type in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_log_backup_interal_actor_acting_duration_sec_count{instance=~\"$instance\"}[$__rate_interval])) by (message)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14912,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Internal Message Handling Duration (P99)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV internal message handling duration (p99) in the Backup Log category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(histogram_quantile(0.99, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14913,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Internal Message Handling Duration (P90)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV internal message handling duration (p90) in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "instance"
    ],
    "promql": [
      "sum(histogram_quantile(0.9, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14271,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Initial Scan RocksDB Throughput ",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV initial scan rocksdb throughput  in the Backup Log category. Helps track RocksDB storage engine performance.",
    "component": "TIKV",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op=~\"read_bytes\"}[$__rate_interval])) BY (op, cf)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14270,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Initial Scan RocksDB Operation ",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV initial scan rocksdb operation  in the Backup Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op!~\"read_bytes\"}[$__rate_interval])) BY (op, cf) > 0"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 14915,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Initial Scanning Trigger Reason",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV initial scanning trigger reason in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(increase(tikv_log_backup_initial_scan_reason{instance=~\"$instance\"}[$__rate_interval])) by (reason)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15176,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region Checkpoint Key Putting",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV region checkpoint key putting in the Backup Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "__rate_interval",
      "instance"
    ],
    "promql": [
      "sum(rate(tikv_log_backup_metadata_key_operation{instance=~\"$instance\"}[$__rate_interval])) by (type)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15544,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request Checkpoint Batch Size",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV request checkpoint batch size in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "__interval"
    ],
    "promql": [
      "sum(increase(tidb_log_backup_advancer_batch_size_bucket{type=\"checkpoint\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15716,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Tick Duration",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV tick duration in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval"
    ],
    "promql": [
      "sum(increase(tidb_log_backup_advancer_tick_duration_sec_bucket{step=~\"tick\"}[$__interval])) by (le)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572666,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Region Checkpoint Failure Reason",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV region checkpoint failure reason in the Backup Log category. Helps detect failures and error patterns.",
    "component": "TIKV",
    "phenomenon": "error_rate",
    "object": "region",
    "percentile": [],
    "variables": [
      "__interval"
    ],
    "promql": [
      "sum(increase(tidb_log_backup_region_request_failure{reason!=\"retryable-scan-region\"}[$__interval])) by (reason)",
      ""
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572665,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Request Result",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV request result in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "request",
    "percentile": [],
    "variables": [
      "__interval"
    ],
    "promql": [
      "sum(increase(tidb_log_backup_region_request[$__interval])) by (result)",
      ""
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15359,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Tick Duration (P99)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV tick duration (p99) in the Backup Log category (P99). Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "sum(histogram_quantile(0.99, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 15360,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Tick Duration (P90)",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV tick duration (p90) in the Backup Log category. Helps identify latency issues and performance degradation.",
    "component": "TIKV",
    "phenomenon": "latency",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(histogram_quantile(0.9, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572733,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Get Region Operation Count",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV get region operation count in the Backup Log category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "TIKV",
    "phenomenon": "io",
    "object": "region",
    "percentile": [],
    "variables": [
      "__rate_interval"
    ],
    "promql": [
      "rate(tidb_log_backup_advancer_tick_duration_sec_count{step=\"get-regions-in-range\"}[$__rate_interval])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 23763572734,
    "dashboard": "Starter-Essential-TiKV-Details-for-Clinic",
    "title": "Try Advance Trigger Time",
    "group": "Backup Log",
    "used_for": "Used for monitoring TiKV try advance trigger time in the Backup Log category.",
    "component": "TIKV",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [
      "__interval"
    ],
    "promql": [
      "increase(tidb_log_backup_advancer_tick_duration_sec_count{step=\"try-advance\"}[$__interval])"
    ],
    "dashboard_file": "tikv-detail.json"
  },
  {
    "panel_id": 1623,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "CPU Usage",
    "group": "",
    "used_for": "Used for monitoring node CPU utilization. Helps identify CPU bottlenecks and resource pressure.",
    "component": "",
    "phenomenon": "cpu",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(process_cpu_seconds_total{namespace=\"tidb-serverless\", pod=~\".*etcd-group.*\"}[1m])) by (pod)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1625,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Memory Usage",
    "group": "",
    "used_for": "Used for monitoring node memory usage. Helps identify memory pressure and potential OOM risks.",
    "component": "",
    "phenomenon": "memory",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg(process_resident_memory_bytes{job=~\".*etcd-group.*\"}) by (instance, job)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1627,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Uptime",
    "group": "",
    "used_for": "Used for monitoring node uptime.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "avg((time() - process_start_time_seconds{job=~\".*etcd-group.*\"})) by (instance, job)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1629,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Disk Read/Write",
    "group": "",
    "used_for": "Used for monitoring node disk read/write. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(irate(container_fs_writes_bytes_total{pod=~\".*etcd-group.*\", namespace=\"tidb-serverless\"}[1m])) by (pod)",
      "sum(irate(container_fs_reads_bytes_total{pod=~\".*etcd-group.*\", namespace=\"tidb-serverless\"}[1m])) by (pod)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1631,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Disk IOPS",
    "group": "",
    "used_for": "Used for monitoring node disk iops. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(\n  irate(container_fs_reads_total{pod=~\".*etcd-group.*\", namespace=\"tidb-serverless\"}[1m]) + \n  irate(container_fs_writes_total{pod=~\".*etcd-group.*\", namespace=\"tidb-serverless\"}[1m])\n) by (pod)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1101,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC DB total size",
    "group": "",
    "used_for": "Used for monitoring node mvcc db total size.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_mvcc_db_total_size_in_bytes{pod=~\".*etcd-group.*\"}",
      "etcd_mvcc_db_total_size_in_use_in_bytes{pod=~\".*etcd-group.*\"}"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1105,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "99% WAL fsync duration",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node 99% wal fsync duration in the ETCD Details category (P99). Helps identify latency issues and performance degradation.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, le))"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1104,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "99% Peer round trip time seconds",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node 99% peer round trip time seconds in the ETCD Details category (P99).",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, To, le))"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1107,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "99% backend commit duration",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node 99% backend commit duration in the ETCD Details category (P99). Helps identify latency issues and performance degradation.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, le))"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1106,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "etcd disk wal fsync rate",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node etcd disk wal fsync rate in the ETCD Details category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_disk_wal_fsync_duration_seconds_count{pod=~\".*etcd-group.*\"}[1m]*60)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1607,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Detailed Backend Commit Durations",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node detailed backend commit durations in the ETCD Details category (P99). Helps identify latency issues and performance degradation.",
    "component": "",
    "phenomenon": "latency",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_rebalance_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, le))",
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_spill_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (instance, le))",
      "histogram_quantile(0.99, sum(rate(etcd_debugging_disk_backend_commit_write_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (instance, le))",
      "rate(etcd_disk_backend_commit_duration_seconds_sum{pod=~\".*etcd-group.*\"}[5m])\n  / rate(etcd_disk_backend_commit_duration_seconds_count{pod=~\".*etcd-group.*\"}[5m])"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1108,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "etcd disk backend commit rate",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node etcd disk backend commit rate in the ETCD Details category. Helps identify disk performance issues.",
    "component": "",
    "phenomenon": "throughput",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_disk_backend_commit_duration_seconds_count{pod=~\".*etcd-group.*\"}[1m]*60)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1611,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "WAL and Snapshot",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node wal and snapshot in the ETCD Details category (P99). Helps track snapshot operations and their impact on performance.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [
      99
    ],
    "variables": [],
    "promql": [
      "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, le))",
      "histogram_quantile(0.99, sum(rate(etcd_snap_fsync_duration_seconds_bucket{pod=~\".*etcd-group.*\"}[5m])) by (pod, le))"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1612,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Lease Operations",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node lease operations in the ETCD Details category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_lease_granted_total{pod=~\".*etcd-group.*\"}[5m])",
      "rate(etcd_debugging_lease_revoked_total{pod=~\".*etcd-group.*\"}[5m])"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1613,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC Operations",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node mvcc operations in the ETCD Details category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_mvcc_put_total{pod=~\".*etcd-group.*\"}[5m])",
      "rate(etcd_debugging_mvcc_range_total{pod=~\".*etcd-group.*\"}[5m])",
      "rate(etcd_debugging_mvcc_delete_total{pod=~\".*etcd-group.*\"}[5m])",
      "rate(etcd_debugging_mvcc_txn_total{pod=~\".*etcd-group.*\"}[5m])"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1608,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Compaction",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node compaction in the ETCD Details category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "increase(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum{pod=~\".*etcd-group.*\"}[5m])",
      "increase(etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{pod=~\".*etcd-group.*\"}[5m])"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1615,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC Total Put Size Growth",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node mvcc total put size growth in the ETCD Details category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "rate(etcd_debugging_mvcc_total_put_size_in_bytes{pod=~\".*etcd-group.*\"}[5m])"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1614,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC Keys Count",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node mvcc keys count in the ETCD Details category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_keys_total{pod=~\".*etcd-group.*\"}[5m]"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1618,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Revision",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node revision in the ETCD Details category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_current_revision{pod=~\".*etcd-group.*\"}",
      "etcd_debugging_mvcc_compact_revision{pod=~\".*etcd-group.*\"}"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1616,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC Client Operations",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node mvcc client operations in the ETCD Details category. Helps identify I/O bottlenecks and disk performance issues.",
    "component": "",
    "phenomenon": "io",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "sum(rate(etcd_server_client_requests_total{pod=~\".*etcd-group.*\"}[5m])) by (client_api_version, type, pod)"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1619,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "Watcher",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node watcher in the ETCD Details category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_watcher_total{pod=~\".*etcd-group.*\"}",
      "etcd_debugging_mvcc_watch_stream_total{pod=~\".*etcd-group.*\"}",
      "etcd_debugging_mvcc_slow_watcher_total{pod=~\".*etcd-group.*\"}"
    ],
    "dashboard_file": "etcd-group.json"
  },
  {
    "panel_id": 1617,
    "dashboard": "Starter-Essential-ETCD-Group-for-Clinic",
    "title": "MVCC Events Counter",
    "group": "ETCD Details",
    "used_for": "Used for monitoring node mvcc events counter in the ETCD Details category.",
    "component": "",
    "phenomenon": "",
    "object": "",
    "percentile": [],
    "variables": [],
    "promql": [
      "etcd_debugging_mvcc_delete_total{pod=~\".*etcd-group.*\"}",
      "etcd_debugging_mvcc_events_total{pod=~\".*etcd-group.*\"}"
    ],
    "dashboard_file": "etcd-group.json"
  }
]