{
  "dashboard_title": "PD-Cluster-Clinic",
  "rows": [
    {
      "title": null,
      "panels": [
        {
          "title": "Storage capacity",
          "subtitle": "The total capacity size of the cluster",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})"
          ]
        },
        {
          "title": "Current storage size",
          "subtitle": "The current storage size of the cluster",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"})"
          ]
        },
        {
          "title": "Current storage used",
          "subtitle": "The current storage size and used ratio of the cluster",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_size\"}) / sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"storage_capacity\"})"
          ]
        },
        {
          "title": "Normal stores",
          "subtitle": "The count of healthy stores",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_up_count\"})"
          ]
        },
        {
          "title": "Number of Regions",
          "subtitle": "The total number of Regions without replicas",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"leader_count\"})"
          ]
        },
        {
          "title": "Current peer count",
          "subtitle": "The current peer count of the cluster",
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region_count\"})"
          ]
        },
        {
          "title": "Abnormal stores",
          "subtitle": null,
          "expressions": [
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_disconnected_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_unhealth_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_low_space_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_down_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_offline_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_tombstone_count\"})",
            "sum(pd_cluster_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_slow_count\"})"
          ]
        },
        {
          "title": "Region health",
          "subtitle": "It records the unusual Regions' count which may include pending peers, down peers, extra peers, offline peers, missing peers or learner peers",
          "expressions": [
            "max(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)",
            "sum(pd_regions_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)"
          ]
        },
        {
          "title": "Leader/Primary",
          "subtitle": "It indicates the current leader/primary of services",
          "expressions": [
            "service_member_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Offline store progress",
          "subtitle": null,
          "expressions": [
            "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"removing\"} > 0)"
          ]
        },
        {
          "title": "Online store progress",
          "subtitle": null,
          "expressions": [
            "avg(pd_cluster_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"preparing\"} > 0)"
          ]
        },
        {
          "title": "Left time",
          "subtitle": null,
          "expressions": [
            "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Current scaling speed",
          "subtitle": null,
          "expressions": [
            "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Placement Rules Status",
          "subtitle": "The current peer count of the cluster",
          "expressions": [
            "sum(pd_rule_manager_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (type)"
          ]
        }
      ]
    },
    {
      "title": "Cluster",
      "panels": [
        {
          "title": "PD scheduler config",
          "subtitle": null,
          "expressions": [
            "pd_config_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Store limit",
          "subtitle": null,
          "expressions": [
            "pd_cluster_store_limit"
          ]
        },
        {
          "title": "Cluster ID",
          "subtitle": null,
          "expressions": [
            "pd_cluster_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Region label isolation level",
          "subtitle": null,
          "expressions": [
            "pd_regions_label_level{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Label distribution",
          "subtitle": null,
          "expressions": [
            "sum(pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (name)",
            "pd_cluster_placement_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Current ID allocation",
          "subtitle": null,
          "expressions": [
            "max(pd_cluster_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"idalloc\"})by(type)"
          ]
        },
        {
          "title": "CPU Usage",
          "subtitle": "PD cpu usage calculated with process cpu running seconds",
          "expressions": [
            "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}[2m])",
            "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}"
          ]
        },
        {
          "title": "Memory Usage",
          "subtitle": "PD  memory usage. ",
          "expressions": [
            "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}",
            "pd_service_memory_quota_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*pd.*\"}"
          ]
        },
        {
          "title": "Uptime",
          "subtitle": "PD uptime since last restart",
          "expressions": [
            "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"})"
          ]
        },
        {
          "title": "Goroutine Count",
          "subtitle": "PD process current goroutines count",
          "expressions": [
            "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\",component=~\".*(pd|tso|scheduling).*\"}"
          ]
        },
        {
          "title": "Current TSO Physcial",
          "subtitle": null,
          "expressions": [
            "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)"
          ]
        },
        {
          "title": "Max TSO gap",
          "subtitle": "The gap between TSO and current time",
          "expressions": [
            "sum(pd_cluster_tso_gap_millionseconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}) by (dc)"
          ]
        },
        {
          "title": "Scheduling Allowance Status",
          "subtitle": "The allowance status of the scheduling.",
          "expressions": [
            "pd_scheduling_allowance_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Current TSO Datetime",
          "subtitle": null,
          "expressions": [
            "max(pd_cluster_tso{type=\"tso\", dc=\"global\"})by(type)"
          ]
        }
      ]
    },
    {
      "title": "Operator",
      "panels": [
        {
          "title": "Schedule operator create",
          "subtitle": "The number of different operators that are newly created",
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"create\"}[2m])*60) by (type)"
          ]
        },
        {
          "title": "Schedule operator check",
          "subtitle": "The number of different operators that have been checked. It mainly checks if the current step is finished; if yes, it returns the next step to be executed.",
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"check\"}[2m])*60) by (type)"
          ]
        },
        {
          "title": "Schedule operator finish",
          "subtitle": "The number of different operators that are finished",
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"finish\"}[2m])*60) by (type)"
          ]
        },
        {
          "title": "Schedule operator timeout",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"timeout\"}[2m])*60) by (type)"
          ]
        },
        {
          "title": "Schedule operator replaced or canceled",
          "subtitle": "The number of different operators that are replaced or canceled",
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"cancel\"}[2m])*60) by (type)",
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", event=\"replace\"}[2m])*60) by (type)"
          ]
        },
        {
          "title": "Schedule operators count by state",
          "subtitle": "The number of operators in different status",
          "expressions": [
            "sum(rate(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (event)"
          ]
        },
        {
          "title": "Operator finish duration",
          "subtitle": "The time consumed when the operator is finished in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))",
            "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))"
          ]
        },
        {
          "title": "Operator step finish duration",
          "subtitle": "The time consumed when the operator step is finished in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))",
            "histogram_quantile(0.95, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type, le))"
          ]
        },
        {
          "title": "Schedulers reach limit",
          "subtitle": "The number of different operators that are failed to be created due to limit configuration",
          "expressions": [
            "sum(rate(pd_schedule_operator_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (type,name)",
            "sum(rate(pd_schedule_operator_exceeded_store_limit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (desc)"
          ]
        },
        {
          "title": "99% operator region size",
          "subtitle": "99% operator region size",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_schedule_operator_region_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (type,le))"
          ]
        }
      ]
    },
    {
      "title": "Statistics - balance",
      "panels": [
        {
          "title": "Store capacity",
          "subtitle": "The capacity size of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_capacity\"}"
          ]
        },
        {
          "title": "Store available",
          "subtitle": "The available capacity size of each TiKV instance",
          "expressions": [
            "{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}"
          ]
        },
        {
          "title": "Store used",
          "subtitle": "The used capacity size of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_used\"}"
          ]
        },
        {
          "title": "Store available ratio",
          "subtitle": "It is equal to Store available capacity size over Store capacity size for each TiKV instance",
          "expressions": [
            "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_available\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_capacity\"}) by (address, store)"
          ]
        },
        {
          "title": "Size amplification",
          "subtitle": "The size amplification, which is equal to Store Region size over Store used capacity size, of each TiKV instance",
          "expressions": [
            "sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}) by (address, store) / sum(pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"store_used\"}) by (address, store) * 2^20"
          ]
        },
        {
          "title": "Store Region score",
          "subtitle": "The Region score of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_score\"}",
            "pd_scheduler_op_influence{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}",
            "pd_scheduler_tolerant_resource{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", scheduler=\"balance-region-scheduler\"}"
          ]
        },
        {
          "title": "Store leader score",
          "subtitle": "The leader score of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_score\"}"
          ]
        },
        {
          "title": "Store Region size",
          "subtitle": "The total Region size of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_size\"}"
          ]
        },
        {
          "title": "Store leader size",
          "subtitle": " \tThe total leader size of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_size\"}"
          ]
        },
        {
          "title": "Store Region count",
          "subtitle": "The Region count of each TiKV instance \t",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"region_count\"}"
          ]
        },
        {
          "title": "Store witness count",
          "subtitle": "The witness count of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"witness_count\"}"
          ]
        },
        {
          "title": "Store leader count",
          "subtitle": "The leader count of each TiKV instance",
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"leader_count\"}"
          ]
        }
      ]
    },
    {
      "title": "Statistics - hot write",
      "panels": [
        {
          "title": "Hot Region's leader distribution",
          "subtitle": "The total number of leader Regions under hot write on each TiKV instance",
          "expressions": [
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_leader\"}"
          ]
        },
        {
          "title": "Hot Region's peer distribution",
          "subtitle": "The total number of Regions which are not leader under hot write on each TiKV instance",
          "expressions": [
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_write_region_as_peer\"}"
          ]
        },
        {
          "title": "Store write rate bytes",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_bytes\"}",
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_bytes\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-region\"}"
          ]
        },
        {
          "title": "Total written keys on hot leader Regions",
          "subtitle": "The total bytes of hot write on leader Regions for each TiKV instance",
          "expressions": [
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_keys_as_leader\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-leader\"}"
          ]
        },
        {
          "title": "Total written bytes on hot leader Regions",
          "subtitle": "The total keys of hot write on leader Regions for each TiKV instance",
          "expressions": [
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"total_write_bytes_as_leader\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-write-leader\"}"
          ]
        },
        {
          "title": "Store write rate keys",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_rate_keys\"}",
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_regions_write_rate_keys\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-write-region\"}"
          ]
        },
        {
          "title": "Store write query",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_write_query_rate\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-write-leader\"}"
          ]
        },
        {
          "title": "Hot cache write entry numbers",
          "subtitle": null,
          "expressions": [
            "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"write\"}"
          ]
        },
        {
          "title": "Selector write events",
          "subtitle": "The select events of hot read scheduler",
          "expressions": [
            "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*write.*\"}[2m])"
          ]
        },
        {
          "title": "Direction of hotspot transfer leader",
          "subtitle": null,
          "expressions": [
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)"
          ]
        },
        {
          "title": "Direction of hotspot move peer",
          "subtitle": null,
          "expressions": [
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"write\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"write\"}[2m]))by (store)",
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"write\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"write\"}[2m]))by (store)"
          ]
        },
        {
          "title": "Remaining items in Queue",
          "subtitle": null,
          "expressions": [
            "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"write\"}"
          ]
        }
      ]
    },
    {
      "title": "Statistics - hot read",
      "panels": [
        {
          "title": "Hot Region's peer distribution",
          "subtitle": "The total number of peer Regions under hot read on each TiKV instance",
          "expressions": [
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_peer\"}",
            "pd_hotspot_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"hot_read_region_as_leader\"}"
          ]
        },
        {
          "title": "Store read rate bytes",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_bytes\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-leader\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-byte-rate-read-region\"}"
          ]
        },
        {
          "title": "Store read rate keys",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_rate_keys\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-leader\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-key-rate-read-region\"}"
          ]
        },
        {
          "title": "Store read query",
          "subtitle": null,
          "expressions": [
            "pd_scheduler_store_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"store_read_query_rate\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-leader\"}",
            "pd_scheduler_hot_peers_summary{type=\"exp-query-rate-read-region\"}"
          ]
        },
        {
          "title": "Direction of hotspot transfer leader",
          "subtitle": null,
          "expressions": [
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"transfer-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)"
          ]
        },
        {
          "title": "Direction of hotspot move peer",
          "subtitle": null,
          "expressions": [
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"out\",rw=\"read\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-peer\",direction=\"in\",rw=\"read\"}[2m]))by (store)",
            "- sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"out\",rw=\"read\"}[2m]))by (store)",
            "sum(delta(pd_scheduler_hot_region_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\",type=\"move-leader\",direction=\"in\",rw=\"read\"}[2m]))by (store)"
          ]
        },
        {
          "title": "Hot cache read entry numbers",
          "subtitle": null,
          "expressions": [
            "pd_hotcache_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=\"total_length\", store=~\"store-$store\", type=\"read\"}"
          ]
        },
        {
          "title": "Remaining items in Queue",
          "subtitle": null,
          "expressions": [
            "pd_hotcache_flow_queue_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"read\"}"
          ]
        },
        {
          "title": "Selector read events",
          "subtitle": "The select events of hot read scheduler",
          "expressions": [
            "rate(pd_scheduler_hot_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=~\".*read.*\"}[2m])"
          ]
        }
      ]
    },
    {
      "title": "Scheduler",
      "panels": [
        {
          "title": "Scheduler is running",
          "subtitle": "The current running schedulers",
          "expressions": [
            "pd_scheduler_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"allow\"}"
          ]
        },
        {
          "title": "Balance leader movement",
          "subtitle": "The leader movement details among TiKV instances",
          "expressions": [
            "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (source)",
            "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[2m])) by (target)"
          ]
        },
        {
          "title": "Balance Region movement",
          "subtitle": "The Region movement details among TiKV instances",
          "expressions": [
            "-sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (source)",
            "sum(delta(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[2m])) by (target)"
          ]
        },
        {
          "title": "Balance leader scheduler",
          "subtitle": "The inner status of balance leader scheduler",
          "expressions": [
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m])) by (name)",
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-leader-scheduler\"}[5m]))"
          ]
        },
        {
          "title": "Balance Region scheduler",
          "subtitle": "The inner status of balance Region scheduler",
          "expressions": [
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m])) by (name)",
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-region-scheduler\"}[5m]))"
          ]
        },
        {
          "title": "Balance Hot Region scheduler",
          "subtitle": "The inner status of balance Hot Region scheduler",
          "expressions": [
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"balance-hot-region-scheduler\"}[5m])) by (name)"
          ]
        },
        {
          "title": "Split Bucket scheduler",
          "subtitle": "The inner status of split bucket scheduler",
          "expressions": [
            "sum(rate(pd_scheduler_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"split-bucket-scheduler\"}[5m])) by (name)"
          ]
        },
        {
          "title": "Balance Direction",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_balance_direction{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60) by (type, source, target)"
          ]
        },
        {
          "title": "Patrol Region time",
          "subtitle": null,
          "expressions": [
            "max(max(pd_checker_patrol_regions_time{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})by(instance))"
          ]
        },
        {
          "title": "Rule checker",
          "subtitle": "The replica checker's status",
          "expressions": [
            "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"rule_checker\"}[2m])) by (name)"
          ]
        },
        {
          "title": "Replica checker",
          "subtitle": "The replica checker's status",
          "expressions": [
            "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"replica_checker\"}[2m])) by (name)"
          ]
        },
        {
          "title": "Region merge checker",
          "subtitle": "The merge checker's status",
          "expressions": [
            "sum(rate(pd_checker_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"merge_checker\"}[2m])) by (name)"
          ]
        },
        {
          "title": "Filter source",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-source\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (source, type, scope)"
          ]
        },
        {
          "title": "Filter target",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", action=\"filter-target\", type!=\"store-state-tombstone-filter\"}[2m])*60) by (target, type, scope)",
            "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"distinct-filter\"}[2m])*60) by (source, target, type, scope)",
            "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-filter\"}[2m])*60) by (source, target, type, scope)",
            "sum(rate(pd_schedule_filter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", action=\"filter-target\",type=\"rule-fit-leader-filter\"}[2m])*60) by (source, target, type, scope)"
          ]
        }
      ]
    },
    {
      "title": "Scatter and Splitter",
      "panels": [
        {
          "title": "scatter operator event",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"skip\"}[2m])*60) by (event)",
            "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fail\"}[2m])*60",
            "rate(pd_schedule_scatter_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"success\"}[2m])*60"
          ]
        },
        {
          "title": "scatter store selection",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"false\"}[2m])*60) by (store)",
            "sum(rate(pd_schedule_scatter_distribution{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", engine=\"tikv\", is_leader=\"true\"}[2m])*60) by (store)"
          ]
        }
      ]
    },
    {
      "title": "gRPC",
      "panels": [
        {
          "title": "gRPC Completed commands rate",
          "subtitle": "The rate of completing each kind of gRPC commands",
          "expressions": [
            "sum(rate(grpc_server_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, grpc_method)"
          ]
        },
        {
          "title": "gRPC 99% Completed commands duration",
          "subtitle": "The time consumed of completing each kind of gRPC commands in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, grpc_method, le))"
          ]
        },
        {
          "title": "gRPC commands concurrency number",
          "subtitle": "The concurrency number of each kind of gRPC commands",
          "expressions": [
            "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"grpc\"}[1m])"
          ]
        }
      ]
    },
    {
      "title": "HTTP",
      "panels": [
        {
          "title": "HTTP Completed commands rate",
          "subtitle": "The rate of completing each kind of HTTP requests",
          "expressions": [
            "sum(rate(pd_service_audit_handling_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,service,ip)"
          ]
        },
        {
          "title": "HTTP 99% Completed commands duration",
          "subtitle": "The time consumed of completing each kind of HTTP request in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_service_audit_handling_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) by (instance, service, le))"
          ]
        },
        {
          "title": "HTTP commands concurrency number",
          "subtitle": "The concurrency number of each kind of HTTP commands",
          "expressions": [
            "max_over_time(pd_server_api_concurrency{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", kind=\"http\"}[1m])"
          ]
        }
      ]
    },
    {
      "title": "etcd",
      "panels": [
        {
          "title": "MVCC DB total size",
          "subtitle": null,
          "expressions": [
            "etcd_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}",
            "etcd_mvcc_db_total_size_in_use_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\".*pd.*\"}"
          ]
        },
        {
          "title": "Handle transactions rate",
          "subtitle": "The rate of handling etcd transactions",
          "expressions": [
            "sum(rate(pd_txn_handle_txns_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result)"
          ]
        },
        {
          "title": "99% Handle transactions duration",
          "subtitle": "The time consumed of handling etcd transactions in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, result, le))"
          ]
        },
        {
          "title": "99% Peer round trip time seconds",
          "subtitle": "The latency of the network in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, To, le))"
          ]
        },
        {
          "title": "99% WAL fsync duration",
          "subtitle": "The time consumed of writing WAL into the persistent storage in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))"
          ]
        },
        {
          "title": "etcd disk wal fsync rate",
          "subtitle": "The rate of writing WAL into the persistent storage",
          "expressions": [
            "rate(etcd_disk_wal_fsync_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"pd\"}[2m])*60"
          ]
        },
        {
          "title": "99% backend commit duration",
          "subtitle": "The time consumed of committing an incremental snapshot of its most recent changes to the persistent storage in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])) by (instance, le))"
          ]
        },
        {
          "title": "etcd disk backend commit rate",
          "subtitle": "The rate of committing an incremental snapshot of its most recent changes to the persistent storage",
          "expressions": [
            "rate(etcd_disk_backend_commit_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60"
          ]
        },
        {
          "title": "99% Endpoint health check latency",
          "subtitle": "The time consumed of etcd endpoint health check in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_server_etcd_endpoint_latency_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}[2m])) by (instance, endpoint, le))"
          ]
        },
        {
          "title": "Endpoint health state",
          "subtitle": "The state of the endpoint health.",
          "expressions": [
            "pd_server_etcd_client{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", source=\"server-etcd-client\"}"
          ]
        },
        {
          "title": "Raft term",
          "subtitle": "The current term of Raft",
          "expressions": [
            "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"term\"}"
          ]
        },
        {
          "title": "Raft committed index",
          "subtitle": "The last committed index of Raft",
          "expressions": [
            "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"committedIndex\"}"
          ]
        },
        {
          "title": "Raft applied index",
          "subtitle": " \tThe last applied index of Raft",
          "expressions": [
            "pd_server_etcd_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"appliedIndex\"}"
          ]
        }
      ]
    },
    {
      "title": "TiDB",
      "panels": [
        {
          "title": "PD server TSO handle duration",
          "subtitle": "The time consumed of handling TiDB requests",
          "expressions": [
            "histogram_quantile(0.90, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.999, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))"
          ]
        },
        {
          "title": "PD client requests handle duration",
          "subtitle": "The time consumed of handling TiDB requests",
          "expressions": [
            "avg(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type) /  avg(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)",
            "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))",
            "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type, le))"
          ]
        },
        {
          "title": "Handle requests count",
          "subtitle": "The count of TSO grpc requests",
          "expressions": [
            "sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)",
            "sum(rate(pd_tso_events{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type,dc)",
            "sum(rate(tso_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)"
          ]
        },
        {
          "title": "Tso Request batch size",
          "subtitle": "The batch size of TiDB tso requests",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))",
            "histogram_quantile(0.90, sum(rate(pd_client_request_handle_tso_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le))",
            "sum(rate(pd_client_request_handle_tso_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) / sum(rate(pd_client_request_handle_tso_batch_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m]))"
          ]
        }
      ]
    },
    {
      "title": "Heartbeat",
      "panels": [
        {
          "title": "Heartbeat region event QPS",
          "subtitle": "The heartbeat update cache or kv OPS",
          "expressions": [
            "sum(rate(pd_cluster_region_event{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, event)"
          ]
        },
        {
          "title": "Region heartbeat handle latency overview",
          "subtitle": "The region heartbeat handle duration by levels",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))",
            "histogram_quantile(0.9, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))",
            "histogram_quantile(0.8, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (le))",
            "sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) / sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m]))"
          ]
        },
        {
          "title": "TiKV side heartbeat statistics",
          "subtitle": "The total number of PD heartbeat messages and pending status",
          "expressions": [
            "sum(rate(tikv_pd_heartbeat_message_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)",
            "tikv_pd_pending_heartbeat_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "99% Region heartbeat handle latency by store",
          "subtitle": "The region heartbeat handle duration in .99 by store",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))"
          ]
        },
        {
          "title": "Region heartbeat report",
          "subtitle": "The count of the heartbeats which each TiKV instance reports to PD",
          "expressions": [
            "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"ok\"}[2m])*60) by (address, store)"
          ]
        },
        {
          "title": "Heartbeat Performance Duration BreakDown (Accumulation)",
          "subtitle": "The breakdown metric about heartbeat",
          "expressions": [
            "sum(rate(pd_core_region_heartbeat_breakdown_handle_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (name)",
            "sum(rate(pd_core_acquire_regions_lock_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (type)"
          ]
        },
        {
          "title": "Concurrent Runner Pending Task",
          "subtitle": "The count of the heartbeats which pending in the task queue.",
          "expressions": [
            "pd_ratelimit_runner_pending_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}",
            "pd_ratelimit_runner_task_max_waiting_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Concurrent Runner Failed Task",
          "subtitle": "The count of the heartbeats which faileds in the task queue.",
          "expressions": [
            "rate(pd_ratelimit_runner_failed_tasks_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])*60"
          ]
        },
        {
          "title": "Region schedule push",
          "subtitle": "The count of the corresponding schedule commands which PD sends to each TiKV instance",
          "expressions": [
            "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\"}[5m])*60) by (address, store, status)"
          ]
        },
        {
          "title": "Region heartbeat report error",
          "subtitle": "The count of the heartbeats with the error status",
          "expressions": [
            "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"err\"}[2m])*60) by (address, store)"
          ]
        },
        {
          "title": "Region heartbeat report active",
          "subtitle": "The count of the heartbeats with the ok status",
          "expressions": [
            "sum(rate(pd_scheduler_region_heartbeat{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"report\", status=\"bind\"}[2m])*60) by (address, store)"
          ]
        },
        {
          "title": "99% store heartbeat handle duration",
          "subtitle": "The store heartbeat handle duration in .99",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_scheduler_handle_store_heartbeat_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))"
          ]
        },
        {
          "title": "Heartbeat stream report",
          "subtitle": "The count of the message which PD send to each TiKV instance",
          "expressions": [
            "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"ok\"}[2m])*60) by (address, store)"
          ]
        },
        {
          "title": "Heartbeat stream report error",
          "subtitle": "The count of the message which PD send to each TiKV instance with error status",
          "expressions": [
            "sum(rate(pd_hbstream_region_message{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\", type=\"push\", status=\"err\"}[2m])*60) by (address, store)"
          ]
        },
        {
          "title": "99% Bucket Report Interval",
          "subtitle": "The Interval of TIkv bucket report interval",
          "expressions": [
            "histogram_quantile(0.99, sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, le))"
          ]
        },
        {
          "title": "Bucket Report State",
          "subtitle": "The State of Bucket Report",
          "expressions": [
            "sum(rate(pd_server_bucket_report{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", store=~\"$store\"}[2m])) by (address, store, type,status)"
          ]
        }
      ]
    },
    {
      "title": "Heartbeat distribution ",
      "panels": [
        {
          "title": "Read Region Byte",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Write Region Byte",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Read Region Key",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Write Region Key",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Store Heartbeat Interval",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Region Heartbeat Interval",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Bucket Report Interval",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        },
        {
          "title": "Hot Degree of Bucket",
          "subtitle": null,
          "expressions": [
            "sum(rate(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[2m])) by (le)"
          ]
        }
      ]
    },
    {
      "title": "Region storage",
      "panels": [
        {
          "title": "Syncer index",
          "subtitle": null,
          "expressions": [
            "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"sync_index\"}"
          ]
        },
        {
          "title": "History last index",
          "subtitle": null,
          "expressions": [
            "pd_region_syncer_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"last_index\"}"
          ]
        }
      ]
    },
    {
      "title": "DR Autosync",
      "panels": [
        {
          "title": "State",
          "subtitle": null,
          "expressions": [
            "max(pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"})"
          ]
        },
        {
          "title": "State of all instances",
          "subtitle": null,
          "expressions": [
            "pd_replication_dr_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Tick",
          "subtitle": null,
          "expressions": [
            "rate(pd_replication_dr_tick_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}[5m])"
          ]
        },
        {
          "title": "State ID",
          "subtitle": null,
          "expressions": [
            "pd_replication_dr_state_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Recovered region",
          "subtitle": null,
          "expressions": [
            "pd_replication_dr_recovered_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        },
        {
          "title": "Recover progress",
          "subtitle": null,
          "expressions": [
            "pd_replication_dr_recover_progress{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\"}"
          ]
        }
      ]
    }
  ]
}