{
  "dashboard_title": "Test-Cluster-TiKV-Details",
  "rows": [
    {
      "title": "Duration",
      "panels": [
        {
          "title": "Write Pipeline Duration",
          "subtitle": "Write Pipeline Composition",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_append_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Cop Read Duration",
          "subtitle": "Read Duration Composition",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by (le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Cluster",
      "panels": [
        {
          "title": "Store size",
          "subtitle": "The storage size per TiKV instance",
          "expressions": [
            "sum((\n    tikv_store_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all_used\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Available size",
          "subtitle": "The available capacity size of each TiKV instance",
          "expressions": [
            "sum((\n    tikv_store_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"available\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Capacity size",
          "subtitle": "The capacity size per TiKV instance",
          "expressions": [
            "sum((\n    tikv_store_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "CPU",
          "subtitle": "The CPU usage of each TiKV instance",
          "expressions": [
            "sum(rate(\n    process_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "((\n    tikv_server_cpu_cores_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  ",
            "max((\n    tikv_server_cpu_cores_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  ",
            "min((\n    tikv_server_cpu_cores_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  "
          ]
        },
        {
          "title": "Memory",
          "subtitle": "The memory usage per TiKV instance",
          "expressions": [
            "sum((\n    process_resident_memory_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n)) by (instance) ",
            "((\n    tikv_server_memory_quota_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  ",
            "max((\n    tikv_server_memory_quota_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  ",
            "min((\n    tikv_server_memory_quota_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  "
          ]
        },
        {
          "title": "IO utilization",
          "subtitle": "The I/O utilization per TiKV instance",
          "expressions": [
            "sum(rate(\n    node_disk_io_time_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "MBps",
          "subtitle": "The total bytes of read and write in each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_engine_flow_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"wal_file_bytes\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_engine_flow_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_in_memory_engine_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "QPS",
          "subtitle": "The number of leaders on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_grpc_msg_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Errps",
          "subtitle": "The total number of the gRPC message failures",
          "expressions": [
            "sum(rate(\n    tikv_grpc_msg_fail_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(delta(\n    tikv_pd_heartbeat_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"noop\"}\n    [$__rate_interval]\n)) by (instance) < 1",
            "sum(rate(\n    tikv_critical_error_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Leader",
          "subtitle": "The number of leaders on each TiKV instance",
          "expressions": [
            "sum((\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Region",
          "subtitle": "The number of Regions and Buckets on each TiKV instance",
          "expressions": [
            "sum((\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"buckets\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Uptime",
          "subtitle": "TiKV uptime since the last restart",
          "expressions": [
            "(time() - ((\n    process_start_time_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    \n))  )"
          ]
        }
      ]
    },
    {
      "title": "Errors",
      "panels": [
        {
          "title": "Critical error",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_critical_error_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Server is busy",
          "subtitle": "\nIndicates occurrences of events that make the TiKV instance unavailable\ntemporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor\nFull",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_channel_full_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) ",
            "sum(rate(\n    tikv_coprocessor_request_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"full\"}\n    [$__rate_interval]\n)) by (instance) ",
            "avg((\n    tikv_engine_write_stall\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write_stall_percentile99\",db=~\"$db\"}\n    \n)) by (instance, db) ",
            "sum(rate(\n    tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum((\n    tikv_raftstore_process_busy\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, type) "
          ]
        },
        {
          "title": "Server report failures",
          "subtitle": "The total number of reporting failure messages",
          "expressions": [
            "sum(rate(\n    tikv_server_report_failure_msg_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, instance, store_id) "
          ]
        },
        {
          "title": "Raftstore error",
          "subtitle": "The number of different raftstore errors on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_storage_engine_async_request_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"success|all\"}\n    [$__rate_interval]\n)) by (instance, status) "
          ]
        },
        {
          "title": "Scheduler error",
          "subtitle": "The number of scheduler errors per type on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"snapshot_err|prepare_write_err\"}\n    [$__rate_interval]\n)) by (instance, stage) "
          ]
        },
        {
          "title": "Coprocessor error",
          "subtitle": "The number of different coprocessor errors on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_request_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, reason) "
          ]
        },
        {
          "title": "gRPC message error",
          "subtitle": "The number of gRPC message errors per type on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_grpc_msg_fail_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Leader drop",
          "subtitle": "The count of dropped leaders per TiKV instance",
          "expressions": [
            "sum(delta(\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Leader missing",
          "subtitle": "The count of missing leaders per TiKV instance",
          "expressions": [
            "sum((\n    tikv_raftstore_leader_missing\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Damaged files",
          "subtitle": "RocksDB damaged SST files",
          "expressions": [
            "((\n    tikv_rocksdb_damaged_files\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  ",
            "((\n    tikv_rocksdb_damaged_files_deleted\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Log Replication Rejected",
          "subtitle": "The count of Log Replication Reject caused by follower memory insufficient",
          "expressions": [
            "sum(rate(\n    tikv_server_raft_append_rejects\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Server",
      "panels": [
        {
          "title": "CF size",
          "subtitle": "The size of each column family",
          "expressions": [
            "sum((\n    tikv_engine_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Channel full",
          "subtitle": "The total number of channel full errors on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_channel_full_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Active written leaders",
          "subtitle": "The number of leaders being written on each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_region_written_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Approximate region size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_raftstore_region_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Approximate region size",
          "subtitle": "The approximate Region size",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_region_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_region_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_region_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_region_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_region_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Huge memtable size",
          "subtitle": "Histogram of huge mem table bytes for regions",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_region_huge_mem_table_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_region_huge_mem_table_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_region_huge_mem_table_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_region_huge_mem_table_bytes_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_region_huge_mem_table_bytes_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Huge l0 table size",
          "subtitle": "Histogram of huge l0 table bytes for regions",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_region_huge_l0_table_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_region_huge_l0_table_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_region_huge_l0_table_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_region_huge_l0_table_bytes_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_region_huge_l0_table_bytes_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Region written bytes",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_region_written_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region average written bytes",
          "subtitle": "The average rate of writing bytes to Regions per TiKV instance",
          "expressions": [
            "(sum(rate(\n    tikv_region_written_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_region_written_bytes_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) )"
          ]
        },
        {
          "title": "Region written keys",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_region_written_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region average written keys",
          "subtitle": "The average rate of written keys to Regions per TiKV instance",
          "expressions": [
            "(sum(rate(\n    tikv_region_written_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_region_written_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) )"
          ]
        },
        {
          "title": "Clear overlap region duration",
          "subtitle": "Bucketed histogram of clear overlap region duration.",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_clear_overlap_region_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_raftstore_clear_overlap_region_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_clear_overlap_region_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Written key size",
          "subtitle": "The key size for every put of apply worker",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_apply_key_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Written value size for every put of apply worker",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_raftstore_apply_value_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Hibernate Peers",
          "subtitle": "The number of peers in hibernated state",
          "expressions": [
            "sum((\n    tikv_raftstore_hibernated_peer_state\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, state) "
          ]
        },
        {
          "title": "Memory trace",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_server_mem_trace_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n    \n))  ",
            "((\n    raft_engine_memory_usage\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Raft Entry Cache Evicts",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_raft_entries_evict_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Resolve address duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_address_resolve_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Thread Pool Schedule Wait Duration$optional_quantile",
          "subtitle": null,
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Average Thread Pool Schedule Wait Duration",
          "subtitle": "The average schedule wait duration for thread pools",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, priority, $additional_groupby)  / sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, priority, $additional_groupby) )",
            "((\n    tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  ",
            "((\n    tikv_tokio_pool_mean_first_poll_delay_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    \n))  "
          ]
        },
        {
          "title": "Disk IO time per second",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_storage_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
            "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Disk IO bytes per second",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_storage_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
            "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "gRPC",
      "panels": [
        {
          "title": "gRPC message count",
          "subtitle": "The count of different kinds of gRPC message",
          "expressions": [
            "sum(rate(\n    tikv_grpc_msg_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_grpc_msg_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (type, priority, $additional_groupby) "
          ]
        },
        {
          "title": "gRPC message failed",
          "subtitle": "The count of different kinds of gRPC message which is failed",
          "expressions": [
            "sum(rate(\n    tikv_grpc_msg_fail_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "gRPC message duration$optional_quantile",
          "subtitle": "The $optional_quantile of execution time of gRPC message",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_grpc_msg_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_grpc_msg_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n    [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Average gRPC message duration",
          "subtitle": "The average execution time of gRPC message",
          "expressions": [
            "(sum(rate(\n    tikv_grpc_msg_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_grpc_msg_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
            "(sum(rate(\n    tikv_grpc_msg_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, priority, $additional_groupby)  / sum(rate(\n    tikv_grpc_msg_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, priority, $additional_groupby) )"
          ]
        },
        {
          "title": "gRPC batch size",
          "subtitle": "The $optional_quantile of execution time of gRPC message",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_grpc_req_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_grpc_resp_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_grpc_req_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_server_grpc_req_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "(sum(rate(\n    tikv_server_grpc_resp_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_server_grpc_resp_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_request_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_request_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_server_request_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )"
          ]
        },
        {
          "title": "raft message batch size",
          "subtitle": null,
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_raft_message_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_raft_message_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_server_raft_message_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )"
          ]
        },
        {
          "title": "gRPC request sources QPS",
          "subtitle": "The QPS of different sources of gRPC request",
          "expressions": [
            "sum(rate(\n    tikv_grpc_request_source_counter_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (source, $additional_groupby) "
          ]
        },
        {
          "title": "gRPC request sources duration",
          "subtitle": "The duration of different sources of gRPC request",
          "expressions": [
            "sum(rate(\n    tikv_grpc_request_source_duration_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (source, $additional_groupby) "
          ]
        },
        {
          "title": "gRPC resource group QPS",
          "subtitle": "The QPS of different resource groups of gRPC request",
          "expressions": [
            "sum(rate(\n    tikv_grpc_resource_group_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) "
          ]
        },
        {
          "title": "gRPC Compression Status for Msg",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_grpc_compression_type_for_msg\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "gRPC Compression Status for Raft Replication",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_grpc_compression_type_for_raft\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Storage",
      "panels": [
        {
          "title": "Storage command total",
          "subtitle": "The total count of different kinds of commands received",
          "expressions": [
            "sum(rate(\n    tikv_storage_command_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Storage async request error",
          "subtitle": "The total number of engine asynchronous request errors",
          "expressions": [
            "sum(rate(\n    tikv_storage_engine_async_request_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"all|success\"}\n    [$__rate_interval]\n)) by (status, $additional_groupby) "
          ]
        },
        {
          "title": "Storage async write duration",
          "subtitle": "The time consumed by processing asynchronous write requests",
          "expressions": [
            "sum(increase(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Storage async write duration",
          "subtitle": "The storage async write duration",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Storage async snapshot duration",
          "subtitle": "The time consumed by processing asynchronous snapshot requests",
          "expressions": [
            "sum(increase(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Storage async snapshot duration",
          "subtitle": "The storage async snapshot duration",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Storage async snapshot duration (pure local read)",
          "subtitle": "The storage async snapshot duration without the involving of raftstore",
          "expressions": [
            "sum(increase(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Storage async snapshot duration (pure local read)",
          "subtitle": "The storage async snapshot duration without the involving of raftstore",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Read index propose wait duration",
          "subtitle": "Read index propose wait duration associated with async snapshot",
          "expressions": [
            "sum(increase(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Read index propose wait duration",
          "subtitle": "Read index propose wait duration associated with async snapshot",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Read index confirm duration",
          "subtitle": "Read index confirm duration associated with async snapshot",
          "expressions": [
            "sum(increase(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Read index confirm duration",
          "subtitle": "Read index confirm duration associated with async snapshot",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Process Stat Cpu Usage",
          "subtitle": "CPU usage measured over a 30 second window",
          "expressions": [
            "sum((\n    tikv_storage_process_stat_cpu_usage\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Full compaction duration seconds",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_full_compact_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_full_compact_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_full_compact_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_full_compact_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_full_compact_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Full compaction pause duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_full_compact_pause_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_full_compact_pause_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_full_compact_pause_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_full_compact_pause_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_full_compact_pause_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Full compaction per-increment duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_full_compact_increment_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_full_compact_increment_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_full_compact_increment_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_full_compact_increment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_full_compact_increment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Concurrency manager max-ts",
          "subtitle": "The max_ts in the concurrency manager",
          "expressions": [
            "tikv_concurrency_manager_max_ts_limit",
            "tikv_concurrency_manager_max_ts"
          ]
        }
      ]
    },
    {
      "title": "Local Reader",
      "panels": [
        {
          "title": "Local reader requests",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
            "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Thread CPU",
      "panels": [
        {
          "title": "Raft store CPU",
          "subtitle": "The CPU utilization of raftstore thread",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Async apply CPU",
          "subtitle": "The CPU utilization of async apply",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance, name) "
          ]
        },
        {
          "title": "Raft async writer CPU",
          "subtitle": "The CPU utilization of raft log writer thread",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "gRPC poll CPU",
          "subtitle": "The CPU utilization of gRPC",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Scheduler pool CPU",
          "subtitle": "The CPU utilization of the scheduler pools",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_background.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Unified read pool CPU",
          "subtitle": "The CPU utilization of the unified read pool",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Background worker CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "GC worker CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Import CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Backup CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "CDC worker CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"cdcwkr.*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"cdc_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "TSO worker CPU",
          "subtitle": "The CPU utilization of raftstore thread",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "IME CPU",
          "subtitle": "The CPU utilization of IME threads",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n    [$__rate_interval]\n)) by (instance, name) "
          ]
        },
        {
          "title": "IA CPU",
          "subtitle": "The CPU utilization of IA threads",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "DFS worker CPU",
          "subtitle": "The CPU utilization of DFS worker threads",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"builtin-dfs.*|memory-fs.*|s3-client.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Cloud worker CPU",
          "subtitle": "The CPU utilization of Cloud worker threads",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker-server.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "KvEngine worker CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Busy Threads (>80%)",
          "subtitle": null,
          "expressions": [
            "topk(20,(\n    sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name!~\"rocksdb.*\"}\n    [$__rate_interval]\n)) by (instance, name) > 0.8\n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "IO Breakdown",
      "panels": [
        {
          "title": "Write IO bytes",
          "subtitle": "The throughput of disk write per IO type",
          "expressions": [
            "sum(rate(\n    tikv_io_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_io_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Read IO bytes",
          "subtitle": "The throughput of disk read per IO type",
          "expressions": [
            "sum(rate(\n    tikv_io_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_io_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "IO threshold",
          "subtitle": "The threshold of disk IOs per priority",
          "expressions": [
            "avg((\n    tikv_rate_limiter_max_bytes_per_sec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Rate Limiter Request Wait Duration",
          "subtitle": "IO rate limiter request wait duration.",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_rate_limiter_request_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_rate_limiter_request_wait_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_rate_limiter_request_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )"
          ]
        }
      ]
    },
    {
      "title": "Raft Waterfall",
      "panels": [
        {
          "title": "Storage async write duration",
          "subtitle": "The time consumed by processing asynchronous write requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_storage_engine_async_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store duration",
          "subtitle": "The store time duration of each request",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Apply duration",
          "subtitle": "The apply time duration of each request",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_apply_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_apply_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_apply_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_apply_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_apply_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store propose wait duration",
          "subtitle": "Time from request scheduling to when it is handled by Raftstore",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store batch wait duration",
          "subtitle": "Time from request scheduling to when a batch of requests is formed and prepared to be proposed to Raft",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store send to write queue duration",
          "subtitle": "Time from request scheduling to just before it is sent to the store writer thread",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store send proposal duration",
          "subtitle": "Time from request scheduling to just before it is sent to followers",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store write kv db end duration",
          "subtitle": "Time from request scheduling to when the batch's snapshot state is written to KV DB",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store before write duration",
          "subtitle": "Time from request scheduling to just before it is written to Raft Engine",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_before_write_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_before_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_before_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store write end duration",
          "subtitle": "Time from request scheduling to when it is written to Raft Engine",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_write_end_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_write_end_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_write_end_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store persist duration",
          "subtitle": "Time from request scheduling to when its associated ready is persisted on the leader",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_persist_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_persist_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_persist_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_persist_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_persist_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store commit but not persist duration",
          "subtitle": "Time from request scheduling to when it is committed; at the time of commit, it has not been persisted on the leader",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Store commit and persist duration",
          "subtitle": "Time from request scheduling to when it is committed; at the time of commit, it has been persisted on the leader",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_raftstore_store_wf_commit_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_wf_commit_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Raft IO",
      "panels": [
        {
          "title": "Process ready duration",
          "subtitle": "The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_raft_process_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Process ready duration per server",
          "subtitle": "The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_raft_process_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_raft_process_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_raft_process_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_raft_process_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_raft_process_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Store write send duration",
          "subtitle": "The duration taken by sending raft logs to channels of followers.",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_store_write_send_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Store write send duration per server",
          "subtitle": "The duration taken by sending raft logs to channels of followers.",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_write_send_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_write_send_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_write_send_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_store_write_send_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_write_send_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Append log duration",
          "subtitle": "The time consumed when Raft appends log",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Append log duration per server",
          "subtitle": "The time consumed when Raft appends log on each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_write_raftdb_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_store_write_raftdb_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_write_raftdb_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Commit log duration",
          "subtitle": "The time consumed when Raft commits log",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Commit log duration per server",
          "subtitle": "The time consumed when Raft commits log on each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_commit_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Apply log duration",
          "subtitle": "The time consumed when Raft applies log",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_apply_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Apply log duration per server",
          "subtitle": "The time consumed for Raft to apply logs per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_apply_log_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Raft Client Wait Connection Ready Duration",
          "subtitle": "The time consumed for Raft Client wait connection ready",
          "expressions": [
            "sum(increase(\n    tikv_server_raft_client_wait_ready_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Raft Client Wait Connection Ready Duration",
          "subtitle": "The time consumed for Raft Client wait connection ready per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_server_raft_client_wait_ready_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (to, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_server_raft_client_wait_ready_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (to, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_raft_client_wait_ready_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (to, $additional_groupby)  / sum(rate(\n    tikv_server_raft_client_wait_ready_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (to, $additional_groupby) )",
            "sum(rate(\n    tikv_server_raft_client_wait_ready_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (to, $additional_groupby) "
          ]
        },
        {
          "title": "Store io task reschedule",
          "subtitle": "The throughput of disk write per IO type",
          "expressions": [
            "sum((\n    tikv_raftstore_io_reschedule_region_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_raftstore_io_reschedule_pending_tasks_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Write task block duration per server $optional_quantile",
          "subtitle": "The time consumed when store write task block on each TiKV instance",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Raft Propose",
      "panels": [
        {
          "title": "Raft proposals per ready",
          "subtitle": "The proposal count of a Regions in a tick",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_apply_proposal_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Raft read/write proposals",
          "subtitle": "The number of proposals per type",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_proposal_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|normal|read_index\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Raft read proposals per server",
          "subtitle": "The number of read proposals which are made by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_proposal_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|read_index\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Raft write proposals per server",
          "subtitle": "The number of write proposals which are made by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_proposal_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"normal\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Raft write apply",
          "subtitle": "The number of write apply",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_write_cmd_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type) "
          ]
        },
        {
          "title": "Raft change_set apply",
          "subtitle": "The number of change_set apply",
          "expressions": [
            "sum(rate(\n    kvengine_apply_change_set_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type) "
          ]
        },
        {
          "title": "Raft change set proposals per server",
          "subtitle": "The number of change set proposals which are made by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_proposal_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Raft change set apply per server",
          "subtitle": "The number of change set apply which are made by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_write_cmd_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set|shard_ingest\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Propose wait duration",
          "subtitle": "The wait time of each proposal",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Propose wait duration per server",
          "subtitle": "The wait time of each proposal in each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_request_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Store write wait duration",
          "subtitle": "The wait time of each store write task",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_store_write_task_wait_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Store write wait duration per server",
          "subtitle": "The wait time of each store write task in each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_store_write_task_wait_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_store_write_task_wait_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_store_write_task_wait_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_store_write_task_wait_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_store_write_task_wait_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Apply wait duration",
          "subtitle": "The wait time of each apply task",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_apply_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Apply wait duration per server",
          "subtitle": "The wait time of each apply task in each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_raftstore_apply_wait_time_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Store write handle msg duration",
          "subtitle": "The handle duration of each store write task msg",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Store write trigger size",
          "subtitle": "The distribution of write trigger size",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_store_write_trigger_wb_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Raft propose speed",
          "subtitle": "The rate at which peers propose logs",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_propose_log_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Perf Context duration",
          "subtitle": "The rate at which peers propose logs",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_store_perf_context_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Raft Process",
      "panels": [
        {
          "title": "Ready handled",
          "subtitle": "The count of different ready type of Raft",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_raft_ready_handled_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_raftstore_raft_process_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Max duration of raft store events",
          "subtitle": "The max time consumed by raftstore events",
          "expressions": [
            "histogram_quantile(0.999999,(\n    sum(rate(\n    tikv_raftstore_event_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.999999,(\n    sum(rate(\n    tikv_broadcast_normal_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "rfstore batch message count",
          "subtitle": "Number of messages polled in one batch by the main peer worker",
          "expressions": [
            "sum(increase(\n    rfstore_receive_msgs_count_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "rfstore batch message size",
          "subtitle": "Size of messages polled in one batch by the main peer worker",
          "expressions": [
            "sum(increase(\n    rfstore_receive_msgs_size_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "rfstore main worker time breakdown",
          "subtitle": "Time spent in different steps of the rfstore main worker loop",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    rfstore_store_msg_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    rfstore_receive_msgs_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    rfstore_sync_aux_worker_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    rfstore_send_aux_task_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    rfstore_process_msgs_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Replica read lock checking duration",
          "subtitle": "Replica read lock checking duration",
          "expressions": [
            "sum(increase(\n    tikv_replica_read_lock_check_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Peer msg length distribution",
          "subtitle": "The length of peer msgs for each round handling",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_peer_msg_len_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Apply msg length distribution",
          "subtitle": "The length of apply msgs for each round handling",
          "expressions": [
            "sum(increase(\n    tikv_raftstore_apply_msg_len_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Enable apply unpersisted log region count",
          "subtitle": "The number of regions that enable apply unpersisted raft log",
          "expressions": [
            "((\n    tikv_raft_enable_unpersisted_apply_regions\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Apply ahead of persistence raft log count",
          "subtitle": "The number of raft logs between apply and persisted index",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raft_apply_ahead_of_persist_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile(1,(\n    sum(rate(\n    tikv_raft_apply_ahead_of_persist_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Raft Message",
      "panels": [
        {
          "title": "Sent messages per server",
          "subtitle": "The number of Raft messages sent by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_raft_sent_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Flush messages per server",
          "subtitle": "The number of Raft messages flushed by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_server_raft_message_flush_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, reason) "
          ]
        },
        {
          "title": "Receive messages per server",
          "subtitle": "The number of Raft messages received by each TiKV instance",
          "expressions": [
            "sum(rate(\n    tikv_server_raft_message_recv_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Messages",
          "subtitle": "The number of different types of Raft messages that are sent",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_raft_sent_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"accept\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Vote",
          "subtitle": "The total number of vote messages that are sent in Raft",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_raft_sent_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"vote\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Raft dropped messages",
          "subtitle": "The number of dropped Raft messages per type",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_raft_dropped_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_raftstore_raft_sent_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"drop\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Raft Message Send Wait duration",
          "subtitle": "The time consumed waiting to send Raft Messages",
          "expressions": [
            "sum(increase(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Raft Message Send Wait Duration",
          "subtitle": "The time consumed waiting to send Raft Messages per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_raft_message_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_server_raft_message_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_server_raft_message_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Raft Message Receive Delay duration",
          "subtitle": "The time consumed to transmit Raft Messages over the network, reported by the receiver",
          "expressions": [
            "sum(increase(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% Raft Message Receive Delay Duration",
          "subtitle": "The time consumed to transmit Raft Messages over the network per TiKV instance, reported by the receiver",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_server_raft_message_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_server_raft_message_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_server_raft_message_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_server_raft_message_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Raft Admin",
      "panels": [
        {
          "title": "Admin proposals",
          "subtitle": "The number of admin proposals",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_proposal_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"conf_change|transfer_leader\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Admin apply",
          "subtitle": "The number of the processed apply command",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_admin_cmd_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"compact\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Check split",
          "subtitle": "The number of raftstore split checks",
          "expressions": [
            "sum(rate(\n    tikv_raftstore_check_split_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"ignore\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "99.99% Check split duration",
          "subtitle": "The time consumed when running split check in .9999",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_raftstore_check_split_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Load base split event",
          "subtitle": null,
          "expressions": [
            "sum(delta(\n    tikv_load_base_split_event\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Load base split duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.8,(\n    sum(rate(\n    tikv_load_base_split_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_load_base_split_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_load_base_split_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_load_base_split_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) )"
          ]
        },
        {
          "title": "Peer in Flashback State",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_raftstore_peer_in_flashback_state\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Raft Log",
      "panels": [
        {
          "title": "Raft log fetch",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    rfstore_raft_entry_fetches_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Raft log async fetch task duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    rfstore_raft_entry_fetches_task_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    rfstore_raft_entry_fetches_task_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    rfstore_raft_entry_fetches_task_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) )",
            "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"async-read-worker\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Raft log lag",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_raftstore_log_lag_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Raft Engine",
      "panels": [
        {
          "title": "Resource Usage",
          "subtitle": "The resource usage of Raft Engine",
          "expressions": [
            "sum((\n    raft_engine_resouce_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Entry Count",
          "subtitle": "The average number of raft log entries",
          "expressions": [
            "avg((\n    raft_engine_total_entries_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Operation",
          "subtitle": "The count of operations per second",
          "expressions": [
            "sum(rate(\n    raft_engine_persist_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    raft_engine_fetch_entries_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Read/Write Duration $optional_quantile",
          "subtitle": "The time used in read/write operation",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_persist_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_fetch_entries_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Flow",
          "subtitle": "The I/O flow rate",
          "expressions": [
            "sum(rate(\n    raft_engine_region_write_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Write Duration Breakdown $optional_quantile",
          "subtitle": "$optional_quantile duration breakdown of write operation",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_wal_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_wal_write_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_apply_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Bytes / Written",
          "subtitle": "The bytes per write",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    raft_engine_region_write_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    raft_engine_region_write_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    raft_engine_region_write_batch_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    raft_engine_region_write_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    raft_engine_region_write_batch_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Duration of WAL Operation - $optional_quantile",
          "subtitle": "$optional_quantile duration breakdown of WAL write operation",
          "expressions": [
            "histogram_quantile(0.999,(\n    sum(rate(\n    raft_engine_truncate_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.999,(\n    sum(rate(\n    raft_engine_rotate_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.999,(\n    sum(rate(\n    raft_engine_compact_wal_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "File Count",
          "subtitle": "The average number of files",
          "expressions": [
            "avg((\n    raft_engine_pending_compaction_wals\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "avg((\n    raft_engine_total_wals_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Other Durations $optional_quantile",
          "subtitle": "The $optional_quantile duration of operations of raft engine",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_take_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    raft_engine_backup_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Raft Log GC Size",
          "subtitle": "The size of GCed raft log",
          "expressions": [
            "avg((\n    raft_engine_log_file_gc_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (status, $additional_groupby) "
          ]
        },
        {
          "title": "Backup Count",
          "subtitle": "The average number of raft engine backup",
          "expressions": [
            "avg((\n    raft_engine_backup_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (status, $additional_groupby) "
          ]
        },
        {
          "title": "DFS Worker Status",
          "subtitle": null,
          "expressions": [
            "((\n    raft_engine_dfs_worker_healthy\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Total Uploaded Bytes to DFS",
          "subtitle": "Total number of bytes uploaded to DFS",
          "expressions": [
            "((\n    raft_engine_dfs_uploaded_bytes_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "DFS Request Rate",
          "subtitle": "The count of DFS requests issued by rfengine.",
          "expressions": [
            "sum(rate(\n    raft_engine_dfs_requests_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "DFS Upload Rate",
          "subtitle": "The byte rates of uploading WAL to DFS.",
          "expressions": [
            "sum(rate(\n    raft_engine_dfs_uploaded_bytes_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Kv Engine",
      "panels": [
        {
          "title": "Memory Usage",
          "subtitle": "The memory usage of Kv Engine",
          "expressions": [
            "max((\n    kv_engine_mem_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\"}\n    \n)) by (type, $additional_groupby) ",
            "max((\n    kv_engine_mem_size_quota_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "max((\n    kv_engine_dfs_load_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "max((\n    kv_engine_ia_manager_segments_memory_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Get operations",
          "subtitle": "The count of get operations",
          "expressions": [
            "sum(rate(\n    kv_engine_get_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Get duration",
          "subtitle": "The time consumed when executing get operations",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_get_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    kv_engine_get_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "avg((\n    kv_engine_get_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Seek operations",
          "subtitle": "The count of seek operations",
          "expressions": [
            "sum(rate(\n    kv_engine_seek_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Seek duration",
          "subtitle": "The time consumed when executing seek operation",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_seek_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    kv_engine_seek_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "avg((\n    kv_engine_seek_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Write operations",
          "subtitle": "The count of write operations",
          "expressions": [
            "sum(rate(\n    kv_engine_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Write duration",
          "subtitle": "The time consumed when executing write operation",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    kv_engine_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "avg((\n    kv_engine_write_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Write flow",
          "subtitle": "The bytes flow of write operations",
          "expressions": [
            "sum(rate(\n    kv_engine_write_flow_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Bytes / Write",
          "subtitle": "The bytes per write",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_write_flow_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_write_flow_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_write_flow_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_write_flow_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_write_flow_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Keys flow",
          "subtitle": "The bytes flow of write operations",
          "expressions": [
            "sum(rate(\n    kv_engine_write_flow_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Keys / Write",
          "subtitle": "The count of keys per write",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_write_flow_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_write_flow_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_write_flow_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_write_flow_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_write_flow_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Block Cache Miss",
          "subtitle": "The miss count of block cache",
          "expressions": [
            "sum(rate(\n    kv_engine_cache_miss\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Shards",
          "subtitle": "The number of shards in KV Engine",
          "expressions": [
            "sum((\n    kv_engine_shards_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_initial_flushed_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_active_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_compacting_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_pending_compaction_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_has_del_prefixes_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_inner_key_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_ready_destroy_range_shards\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Files Count",
          "subtitle": "The number of files in KV Engine",
          "expressions": [
            "sum((\n    kv_engine_cf_level_num_files\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (cf, level, $additional_groupby) "
          ]
        },
        {
          "title": "Files Size",
          "subtitle": "The size of files in KV Engine",
          "expressions": [
            "sum((\n    kv_engine_cf_level_total_sizes_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (cf, level, $additional_groupby) "
          ]
        },
        {
          "title": "Memtable Count",
          "subtitle": "The count of memtables",
          "expressions": [
            "sum((\n    kv_engine_mem_tables_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Memtable Size",
          "subtitle": "The size of memtables",
          "expressions": [
            "sum((\n    kv_engine_mem_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\",type=\"memtable\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "L0 Table Count",
          "subtitle": "The count of L0 tables",
          "expressions": [
            "sum((\n    kv_engine_l0_tables_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "L0 Table Size",
          "subtitle": "The size of L0 tables",
          "expressions": [
            "sum((\n    kv_engine_l0_tables_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Blob Table Count",
          "subtitle": "The count of blob tables",
          "expressions": [
            "sum((\n    kv_engine_blob_tables_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Blob Table Size",
          "subtitle": "The size of blob tables",
          "expressions": [
            "sum((\n    kv_engine_in_use_blob_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, $additional_groupby) ",
            "sum((\n    kv_engine_total_blob_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Partial File Count",
          "subtitle": "The count of partial files",
          "expressions": [
            "sum((\n    kv_engine_partial_l0_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_partial_blob_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_partial_ln_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Index & Filter Size",
          "subtitle": "The size of index and filter files",
          "expressions": [
            "sum((\n    kv_engine_index_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_filter_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Entries Count",
          "subtitle": "The count of entries",
          "expressions": [
            "sum((\n    kv_engine_entries\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_old_entries\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    kv_engine_tombstones\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "DFS Throughput",
          "subtitle": "The throughput of DFS operations",
          "expressions": [
            "sum(rate(\n    kv_engine_dfs_throughput_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",dfs_type=\"s3\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Level Write Throughput",
          "subtitle": "The throughput of load file at each level",
          "expressions": [
            "sum(rate(\n    kv_engine_level_write_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (level, $additional_groupby) "
          ]
        },
        {
          "title": "DFS Request Rate",
          "subtitle": "The OPS of DFS operations",
          "expressions": [
            "sum(rate(\n    kv_engine_dfs_request_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "DFS Request Retry Rate",
          "subtitle": "The OPS of DFS request retries",
          "expressions": [
            "sum(rate(\n    kv_engine_dfs_retry_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "DFS Request Duration",
          "subtitle": "The duration of DFS operations in ms",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_dfs_latency_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_dfs_latency_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_dfs_latency_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_dfs_latency_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_dfs_latency_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "DFS Request Duration with Retry",
          "subtitle": "The duration of DFS operations including retry times in ms",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_dfs_latency_with_retry_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_dfs_latency_with_retry_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_dfs_latency_with_retry_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_dfs_latency_with_retry_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_dfs_latency_with_retry_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "DFS Load Memory Wait Duration",
          "subtitle": "Wait duration when acquiring DFS load memory (only recorded when memory is not immediately available)",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_dfs_load_memory_wait_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    kv_engine_dfs_load_memory_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    kv_engine_dfs_load_memory_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Prepare ChangeSet File Load Source",
          "subtitle": "The file load statistic when preparing changeset",
          "expressions": [
            "sum(rate(\n    kv_engine_prepare_load_remote_file\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    kv_engine_prepare_use_local_file\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "CacheFS Hit",
          "subtitle": "Count of CacheFS hit",
          "expressions": [
            "sum(rate(\n    kv_engine_cachefs_req_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "CacheFS Hit Rate",
          "subtitle": "CacheFS hit rate",
          "expressions": [
            "(sum(rate(\n    kv_engine_cachefs_req_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    kv_engine_cachefs_req_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )"
          ]
        },
        {
          "title": "Prepare ChangeSet File Load Source",
          "subtitle": "The file load statistic when preparing changeset",
          "expressions": [
            "sum(rate(\n    kv_engine_prepare_load_remote_file\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    kv_engine_prepare_use_local_file\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Ingest Level Distribution",
          "subtitle": "The distribution of ingestion levels (0-3) in the storage engine",
          "expressions": [
            "sum(increase(\n    kv_engine_ingest_level_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Ingest Level Rate",
          "subtitle": "The rate of ingestion operations by level",
          "expressions": [
            "sum(rate(\n    kv_engine_ingest_level_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "IA Segment Memory Cache",
          "subtitle": null,
          "expressions": [
            "max((\n    kv_engine_ia_manager_segments_memory_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "max((\n    kv_engine_ia_small_queue_capacity\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "IA Segment Disk Cache",
          "subtitle": null,
          "expressions": [
            "max((\n    kv_engine_ia_manager_segments_disk_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "max((\n    kv_engine_ia_main_queue_capacity\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "IA Segment Cache Hit",
          "subtitle": "Count of IA segment hit",
          "expressions": [
            "(sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  - sum(rate(\n    kv_engine_ia_read_segment_cache_miss\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) )"
          ]
        },
        {
          "title": "IA Segment Cache Hit Rate",
          "subtitle": "IA Segment hit rate",
          "expressions": [
            "(1 - (sum(rate(\n    kv_engine_ia_read_segment_cache_miss\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) ))"
          ]
        },
        {
          "title": "IA Segmenet Read Duration",
          "subtitle": "The time taken by KvEngine to complete read a IA segment",
          "expressions": [
            "sum(increase(\n    kv_engine_ia_read_segment_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% IA segment read duration",
          "subtitle": "The time taken by KvEngine to complete read a IA segment",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    kv_engine_ia_read_segment_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Storage Size by Storage Class",
          "subtitle": "Storage size used by each storage class (IA and Standard)",
          "expressions": [
            "sum((\n    tikv_store_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"used\"}\n    \n)) by (storage_class, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "In Memory Engine",
      "panels": [
        {
          "title": "OPS",
          "subtitle": "Operation per second for cf",
          "expressions": [
            "sum(rate(\n    tikv_in_memory_engine_kv_operations\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type, $additional_groupby) "
          ]
        },
        {
          "title": "Read MBps",
          "subtitle": "The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)",
          "expressions": [
            "sum(rate(\n    tikv_engine_flow_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_in_memory_engine_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Coprocessor Handle duration",
          "subtitle": "The time consumed when handling coprocessor requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Region Cache Hit",
          "subtitle": "Count of region cache hit",
          "expressions": [
            "sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Region Cache Hit Rate",
          "subtitle": "Region cache hit rate",
          "expressions": [
            "(sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_memory_engine\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_snapshot_type_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )"
          ]
        },
        {
          "title": "Region Cache Miss Reason",
          "subtitle": "Reasons for region cache miss",
          "expressions": [
            "sum(rate(\n    tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Memory Usage",
          "subtitle": "The memory usage of the in-memory engine",
          "expressions": [
            "avg((\n    tikv_in_memory_engine_memory_usage_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Region Count",
          "subtitle": "The count of different types of region",
          "expressions": [
            "avg((\n    tikv_in_memory_engine_cache_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, type) "
          ]
        },
        {
          "title": "GC Filter",
          "subtitle": "Rang cache engine garbage collection information",
          "expressions": [
            "sum(rate(\n    tikv_in_memory_engine_gc_filtered\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Region GC Duration",
          "subtitle": "The handle duration of region gc",
          "expressions": [
            "sum(increase(\n    tikv_in_memory_engine_gc_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region Load Duration",
          "subtitle": "The handle duration of region load",
          "expressions": [
            "sum(increase(\n    tikv_in_memory_engine_load_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region Load Count",
          "subtitle": "The count of region loading per seconds",
          "expressions": [
            "sum(delta(\n    tikv_in_memory_engine_load_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Region Eviction Duration",
          "subtitle": "The handle duration of region eviction",
          "expressions": [
            "sum(increase(\n    tikv_in_memory_engine_eviction_duration_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region Eviction Count",
          "subtitle": "The count of region eviction per seconds",
          "expressions": [
            "sum(delta(\n    tikv_in_memory_engine_eviction_duration_secs_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Write duration",
          "subtitle": "The time consumed of write in region cache engine",
          "expressions": [
            "sum(increase(\n    tikv_in_memory_engine_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% In-memory engine write duration per server",
          "subtitle": "The time consumed of write in region cache engine per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_in_memory_engine_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_in_memory_engine_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_in_memory_engine_write_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_in_memory_engine_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_in_memory_engine_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Prepare for write duration",
          "subtitle": "The time consumed of prepare for write in the in-memory engine",
          "expressions": [
            "sum(increase(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "99% In-memory engine prepare for write duration per server",
          "subtitle": "The time consumed of prepare for write in the in-memory engine per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby)  / sum(rate(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) )",
            "sum(rate(\n    tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Iterator operations",
          "subtitle": "The count of different type of iteration operations",
          "expressions": [
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_seek\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_seek_found\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_next\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_next_found\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_prev\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_in_memory_engine_locate\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"number_db_prev_found\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Seek duration",
          "subtitle": "The time consumed when executing seek operation",
          "expressions": [
            "histogram_quantile(1,(\n    sum(rate(\n    tikv_in_memory_engine_seek_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_in_memory_engine_seek_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.95,(\n    sum(rate(\n    tikv_in_memory_engine_seek_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_in_memory_engine_seek_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_in_memory_engine_seek_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )"
          ]
        },
        {
          "title": "Oldest Auto GC SafePoint",
          "subtitle": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one",
          "expressions": [
            "max((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)"
          ]
        },
        {
          "title": "Newest Auto GC SafePoint",
          "subtitle": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one",
          "expressions": [
            "max((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)"
          ]
        },
        {
          "title": "Auto GC SafePoint Gap",
          "subtitle": "The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine",
          "expressions": [
            "(sum((\n    tikv_in_memory_engine_newest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18) - sum((\n    tikv_in_memory_engine_oldest_safe_point\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18))"
          ]
        },
        {
          "title": "Auto GC SafePoint Gap With TiKV",
          "subtitle": "The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point",
          "expressions": [
            "max((\n    tikv_safe_point_gap_with_in_memory_engine\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    \n)) by (instance, $additional_groupby) / (2^18)"
          ]
        }
      ]
    },
    {
      "title": "Cloud Worker Service",
      "panels": [
        {
          "title": "Compaction Request duration",
          "subtitle": "The time consumed to handle remote compaction requests",
          "expressions": [
            "sum(increase(\n    tikv_worker_remote_compact_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Compaction Request duration",
          "subtitle": "The time consumed to handle remote compaction requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_worker_remote_compact_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_worker_remote_compact_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_worker_remote_compact_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_worker_remote_compact_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_worker_remote_compact_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Remote Processing Requests",
          "subtitle": "Number of remote compaction or coprocessor requests are under processing",
          "expressions": [
            "sum((\n    tikv_worker_remote_compact_processing_requests_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    \n)) by ($additional_groupby) ",
            "sum((\n    tikv_worker_remote_cop_processing_requests_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Remote Requests Failures",
          "subtitle": "The number of remote requests that failed to acquire permit, by failure reason",
          "expressions": [
            "sum(rate(\n    tikv_worker_remote_compact_failed_requests_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    tikv_worker_remote_cop_failed_requests_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Concurrency Limiter Waiting Length",
          "subtitle": "Number of remote requests being waiting, limited by concurrency limiter",
          "expressions": [
            "sum((\n    tikv_worker_limiter_waiting_requests_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Concurrency Limiter Waiting Duration",
          "subtitle": "The waiting duration of handling remote requests limited by concurrency limiter",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_worker_limiter_request_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Snapshot Request duration",
          "subtitle": "The time consumed to handle remote copr snapshot duration",
          "expressions": [
            "sum(increase(\n    tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Snapshot Request duration",
          "subtitle": "The time consumed to handle remote copr snapshot duration",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_worker_remote_cop_snapshot_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_worker_remote_cop_snapshot_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_worker_remote_cop_snapshot_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Copr Request Process duration",
          "subtitle": "The time consumed to handle remote copr process duration",
          "expressions": [
            "sum(increase(\n    tikv_worker_remote_cop_process_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Copr Request Process duration",
          "subtitle": "The time consumed to handle remote copr process duration",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_worker_remote_cop_process_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_worker_remote_cop_process_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_worker_remote_cop_process_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_worker_remote_cop_process_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_worker_remote_cop_process_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Copr Request duration",
          "subtitle": "The time consumed to handle remote coprocessor read requests",
          "expressions": [
            "sum(increase(\n    tikv_worker_remote_cop_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Copr Request duration",
          "subtitle": "The time consumed to handle remote coprocessor read requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_worker_remote_cop_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_worker_remote_cop_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_worker_remote_cop_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_worker_remote_cop_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_worker_remote_cop_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Remote Copr Response Size",
          "subtitle": "The size of remote coprocessor response",
          "expressions": [
            "sum(rate(\n    tikv_worker_remote_cop_dag_response_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_worker_remote_analyze_response_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_worker_remote_checksum_response_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Remote Copr Request Operations",
          "subtitle": "The qps of remote coprocessor requests",
          "expressions": [
            "sum(rate(\n    tikv_worker_remote_cop_dag_request_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_worker_remote_analyze_request_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) ",
            "sum(rate(\n    tikv_worker_remote_checksum_request_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Memory Limiter Usage",
          "subtitle": "Number of bytes are currently being used by requests limited by memory limiter",
          "expressions": [
            "sum((\n    tikv_worker_memory_limiter_current_used\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Native BR RPC Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_worker_native_br_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_worker_native_br_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Native BR RPC Operations",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_worker_native_br_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Native BR Backup Operations",
          "subtitle": "Number of successful and failed backup operations",
          "expressions": [
            "sum(rate(\n    native_br_backup_success\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) ",
            "sum(rate(\n    native_br_backup_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Native BR Restore Errors",
          "subtitle": "Number of errors during restoration by type",
          "expressions": [
            "sum(rate(\n    native_br_restore_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum(rate(\n    native_br_restore_rfengine_wal_epoch_overwritten_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Native BR Restored Snapshot Bytes",
          "subtitle": "Total bytes of restored snapshots",
          "expressions": [
            "sum(rate(\n    native_br_restored_snapshot_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Native BR Restore Snapshot Request Duration",
          "subtitle": "Duration of restore snapshot of a shard",
          "expressions": [
            "sum(increase(\n    native_br_restore_snapshot_request_secs_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        }
      ]
    },
    {
      "title": "Local File GC",
      "panels": [
        {
          "title": "On Disk Files",
          "subtitle": "The number of files on disk",
          "expressions": [
            "sum((\n    rfstore_local_file_on_disk\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Alive Files",
          "subtitle": "The number of alive files used by kvengine",
          "expressions": [
            "sum((\n    rfstore_local_file_alive\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "GC Files",
          "subtitle": "The number of files garbage collected",
          "expressions": [
            "sum(rate(\n    rfstore_local_file_gc_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Skip GC Files",
          "subtitle": "The number of files skipped by GC",
          "expressions": [
            "sum((\n    rfstore_local_file_skip_gc\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Pending GC Files",
          "subtitle": "The number of files pending for garbage collection",
          "expressions": [
            "sum((\n    rfstore_local_file_pending_gc\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "GC Errors",
          "subtitle": "The number of local file GC errors",
          "expressions": [
            "sum(rate(\n    rfstore_local_file_error_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Flow Control",
      "panels": [
        {
          "title": "Throttling counter",
          "subtitle": "Counter when throttle happens per limiter",
          "expressions": [
            "sum(rate(\n    kv_engine_limiter_throttle_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (limiter, $additional_groupby) "
          ]
        },
        {
          "title": "Throttling algorithm actions",
          "subtitle": "Counter when different throttling actions happen per limiter",
          "expressions": [
            "sum(rate(\n    kv_engine_limiter_throttle_action_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (limiter, type, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler flow",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_scheduler_write_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_scheduler_throttle_flow\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance)  != 0"
          ]
        },
        {
          "title": "Throttle duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_scheduler_throttle_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Txn command throttled duration",
          "subtitle": "Throttle time for txn storage commands in 1 minute.",
          "expressions": [
            "sum(rate(\n    tikv_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Non-txn command throttled duration",
          "subtitle": "Throttle time for non-txn related processing like analyze or dag in 1 minute.",
          "expressions": [
            "sum(rate(\n    tikv_non_txn_command_throttle_time_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Scheduler",
      "panels": [
        {
          "title": "Scheduler stage total",
          "subtitle": "The total number of commands on each stage",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) ",
            "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler priority commands",
          "subtitle": "The count of different priority commands",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_commands_pri_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (priority, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler pending commands",
          "subtitle": "The count of pending commands per TiKV instance",
          "expressions": [
            "sum((\n    tikv_scheduler_context_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Scheduler running tasks",
          "subtitle": "Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)",
          "expressions": [
            "sum((\n    tikv_scheduler_pool_running_tasks\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Scheduler writing bytes",
          "subtitle": "The total writing bytes of commands on each stage",
          "expressions": [
            "sum((\n    tikv_scheduler_writing_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Scheduler memory quota",
          "subtitle": "The number of bytes used by scheduler",
          "expressions": [
            "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_use\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_scheduler_memory_quota_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Scheduler Pool Wait Duration",
          "subtitle": "Time tasks wait to be scheduled (YATP or Tokio backend)",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance) )",
            "((\n    tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Scheduler Worker Pool",
      "panels": [
        {
          "title": "Schedule Wait Duration",
          "subtitle": "Time from task spawn to first poll",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance) )",
            "((\n    tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Running Tasks",
          "subtitle": "Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)",
          "expressions": [
            "sum(avg_over_time(\n    tikv_scheduler_pool_running_tasks\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (instance, $additional_groupby) "
          ]
        },
        {
          "title": "Task Poll Duration",
          "subtitle": "Time tasks spend executing (being polled)",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_task_poll_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_yatp_task_poll_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n    [$__rate_interval]\n)) by (instance) )",
            "((\n    tikv_scheduler_tokio_pool_mean_poll_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Task Scheduled Duration",
          "subtitle": "Time tasks wait to be polled after being woken",
          "expressions": [
            "((\n    tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Task Idle Duration",
          "subtitle": "Time tasks spend idle waiting on external events (I/O)",
          "expressions": [
            "((\n    tikv_scheduler_tokio_pool_mean_idle_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "First Poll Count",
          "subtitle": "Rate of tasks polled for first time",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_tokio_pool_first_poll_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Scheduler - $command",
      "panels": [
        {
          "title": "Scheduler stage total",
          "subtitle": "The total number of commands on each stage in commit command",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_too_busy_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_scheduler_stage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (stage, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler command duration",
          "subtitle": "The time consumed when executing commit command",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_scheduler_command_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_scheduler_command_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_scheduler_command_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_scheduler_command_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_scheduler_command_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Scheduler latch wait duration",
          "subtitle": "The time which is caused by latch wait in commit command",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_scheduler_latch_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_scheduler_latch_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_scheduler_latch_wait_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_scheduler_latch_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_scheduler_latch_wait_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Scheduler keys read",
          "subtitle": "The count of keys read by a commit command",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_scheduler_kv_command_key_read_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_scheduler_kv_command_key_read_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_scheduler_kv_command_key_read_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_scheduler_kv_command_key_read_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_scheduler_kv_command_key_read_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Scheduler keys written",
          "subtitle": "The count of keys written by a commit command",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_scheduler_kv_command_key_write_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_scheduler_kv_command_key_write_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_scheduler_kv_command_key_write_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_scheduler_kv_command_key_write_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_scheduler_kv_command_key_write_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Scheduler scan details",
          "subtitle": "The keys scan details of each CF when executing commit command",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_kv_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler scan details [lock]",
          "subtitle": "The keys scan details of lock CF when executing commit command",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_kv_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"lock\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler scan details [write]",
          "subtitle": "The keys scan details of write CF when executing commit command",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_kv_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"write\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler scan details [default]",
          "subtitle": "The keys scan details of default CF when executing commit command",
          "expressions": [
            "sum(rate(\n    tikv_scheduler_kv_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"default\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Scheduler command read duration",
          "subtitle": "The time consumed on reading when executing commit command",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_scheduler_processing_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_scheduler_processing_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_scheduler_processing_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_scheduler_processing_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_scheduler_processing_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Check memory locks duration",
          "subtitle": "The time consumed on checking memory locks",
          "expressions": [
            "sum(increase(\n    tikv_storage_check_mem_lock_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        }
      ]
    },
    {
      "title": "Coprocessor Overview",
      "panels": [
        {
          "title": "Request duration",
          "subtitle": "The time consumed to handle coprocessor read requests",
          "expressions": [
            "sum(increase(\n    tikv_coprocessor_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Request duration",
          "subtitle": "The time consumed to handle coprocessor read requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Total Requests",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Total Request Errors",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_request_error\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (reason, $additional_groupby) "
          ]
        },
        {
          "title": "KV Cursor Operations",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "KV Cursor Operations",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_scan_keys_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_scan_keys_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_scan_keys_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Total RocksDB Perf Statistics",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"internal_delete_skipped_count\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Total Response Size",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_response_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Memory Quota",
          "subtitle": "Total bytes of memory used by coprocessor requests",
          "expressions": [
            "sum((\n    tikv_coprocessor_memory_quota\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, type) "
          ]
        }
      ]
    },
    {
      "title": "Coprocessor Detail",
      "panels": [
        {
          "title": "Handle duration",
          "subtitle": "The time consumed when handling coprocessor requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Handle duration by store",
          "subtitle": "The time consumed to handle coprocessor requests per TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_handle_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_handle_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, instance) )",
            "sum(rate(\n    tikv_coprocessor_request_handle_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (req, instance) "
          ]
        },
        {
          "title": "Wait duration",
          "subtitle": "The time consumed when coprocessor requests are wait for being handled",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Wait duration by store",
          "subtitle": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_wait_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance, le) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_wait_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance)  / sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) )",
            "sum(rate(\n    tikv_coprocessor_request_wait_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n    [$__rate_interval]\n)) by (req, instance) "
          ]
        },
        {
          "title": "Total DAG Requests",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_dag_request_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (vec_type, $additional_groupby) "
          ]
        },
        {
          "title": "Total DAG Executors",
          "subtitle": "The total number of DAG executors",
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_executor_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Total Ops Details (Table Scan)",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Total Ops Details (Index Scan)",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (tag, $additional_groupby) "
          ]
        },
        {
          "title": "Total Ops Details by CF (Table Scan)",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) "
          ]
        },
        {
          "title": "Total Ops Details by CF (Index Scan)",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_scan_details\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n    [$__rate_interval]\n)) by (cf, tag, $additional_groupby) "
          ]
        },
        {
          "title": "Memory lock checking duration",
          "subtitle": "The time consumed on checking memory locks for coprocessor requests",
          "expressions": [
            "sum(increase(\n    tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Memory lock checking duration",
          "subtitle": "The time consumed on checking memory locks for coprocessor requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_mem_lock_check_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_coprocessor_mem_lock_check_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_mem_lock_check_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Unified Read Pool",
      "panels": [
        {
          "title": "Schedule Wait Duration",
          "subtitle": "Time from task spawn to first poll",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_yatp_pool_schedule_wait_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n    [$__rate_interval]\n)) by (instance) )",
            "((\n    tikv_tokio_pool_mean_first_poll_delay_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    \n))  "
          ]
        },
        {
          "title": "Running Tasks",
          "subtitle": "Number of concurrently running tasks",
          "expressions": [
            "sum(avg_over_time(\n    tikv_unified_read_pool_running_tasks\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (priority, $additional_groupby) "
          ]
        },
        {
          "title": "Task Poll Duration",
          "subtitle": "Time tasks spend executing (being polled)",
          "expressions": [
            "(sum(rate(\n    tikv_yatp_task_poll_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_yatp_task_poll_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n    [$__rate_interval]\n)) by (instance) )",
            "((\n    tikv_tokio_pool_mean_poll_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    \n))  "
          ]
        },
        {
          "title": "Task Scheduled Duration",
          "subtitle": "Time tasks wait to be polled after being woken",
          "expressions": [
            "((\n    tikv_tokio_pool_mean_scheduled_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    \n))  "
          ]
        },
        {
          "title": "Task Idle Duration",
          "subtitle": "Time tasks spend idle waiting on external events (I/O)",
          "expressions": [
            "((\n    tikv_tokio_pool_mean_idle_duration_seconds\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    \n))  "
          ]
        },
        {
          "title": "First Poll Count",
          "subtitle": "Rate of tasks polled for first time",
          "expressions": [
            "sum(rate(\n    tikv_tokio_pool_first_poll_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "GC",
      "panels": [
        {
          "title": "GC tasks",
          "subtitle": "The count of GC tasks processed by gc_worker",
          "expressions": [
            "sum(rate(\n    tikv_gcworker_gc_tasks_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (task, $additional_groupby) ",
            "sum(rate(\n    tikv_storage_gc_skipped_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (task, $additional_groupby) ",
            "sum(rate(\n    tikv_gcworker_gc_task_fail_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (task, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_worker_too_busy\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "GC tasks duration",
          "subtitle": "The time consumed when executing GC tasks",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_gcworker_gc_task_duration_vec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_gcworker_gc_task_duration_vec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_gcworker_gc_task_duration_vec_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_gcworker_gc_task_duration_vec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_gcworker_gc_task_duration_vec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "TiDB GC seconds",
          "subtitle": "The GC duration",
          "expressions": [
            "histogram_quantile(1,(\n    sum(rate(\n    tidb_tikvclient_gc_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (instance, stage, keyspace_name, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "TiDB GC worker actions",
          "subtitle": "The count of TiDB GC worker actions",
          "expressions": [
            "sum(rate(\n    tidb_tikvclient_gc_worker_actions_total\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n    [$__rate_interval]\n)) by (type, keyspace_name, $additional_groupby) "
          ]
        },
        {
          "title": "ResolveLocks Progress",
          "subtitle": "Progress of ResolveLocks, the first phase of GC",
          "expressions": [
            "max((\n    tidb_tikvclient_range_task_stats\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n    \n)) by (result, type, keyspace_name, $additional_groupby) "
          ]
        },
        {
          "title": "TiKV Auto GC Progress",
          "subtitle": "Progress of TiKV's GC",
          "expressions": [
            "(sum((\n    tikv_gcworker_autogc_processed_regions\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n    \n)) by (instance)  / sum((\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n    \n)) by (instance) )"
          ]
        },
        {
          "title": "GC speed",
          "subtitle": "keys / second",
          "expressions": [
            "sum(rate(\n    tikv_storage_mvcc_gc_delete_versions_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) "
          ]
        },
        {
          "title": "TiKV Auto GC SafePoint",
          "subtitle": "SafePoint used for TiKV's Auto GC",
          "expressions": [
            "max((\n    tikv_gcworker_autogc_safe_point\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n    \n)) by (instance, keyspace_name, $additional_groupby) / (2^18)"
          ]
        },
        {
          "title": "GC lifetime",
          "subtitle": "The lifetime of TiDB GC",
          "expressions": [
            "max((\n    tidb_tikvclient_gc_config\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n    \n)) by (keyspace_name) "
          ]
        },
        {
          "title": "GC interval",
          "subtitle": "The interval of TiDB GC",
          "expressions": [
            "max((\n    tidb_tikvclient_gc_config\n    {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n    \n)) by (keyspace_name) "
          ]
        },
        {
          "title": "GC in Compaction Filter",
          "subtitle": "Keys handled in GC compaction filter",
          "expressions": [
            "sum(rate(\n    tikv_gc_compaction_filtered\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_skip\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_mvcc_rollback\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_orphan_versions\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_perform\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_failure\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_mvcc_deletion_met\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_mvcc_deletion_handled\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) ",
            "sum(rate(\n    tikv_gc_compaction_filter_mvcc_deletion_wasted\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (key_mode, $additional_groupby) "
          ]
        },
        {
          "title": "GC scan write details",
          "subtitle": "GC scan write details",
          "expressions": [
            "sum(rate(\n    tikv_gcworker_gc_keys\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n    [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) "
          ]
        },
        {
          "title": "GC scan default details",
          "subtitle": "GC scan default details",
          "expressions": [
            "sum(rate(\n    tikv_gcworker_gc_keys\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n    [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Pessimistic Locking",
      "panels": [
        {
          "title": "Lock Manager Thread CPU",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"waiter_manager.*\"}\n    [$__rate_interval]\n)) by (instance, name) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"deadlock_detect.*\"}\n    [$__rate_interval]\n)) by (instance, name) "
          ]
        },
        {
          "title": "Lock Manager Handled tasks",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_lock_manager_task_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Waiter lifetime duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_lock_manager_waiter_lifetime_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_lock_manager_waiter_lifetime_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_lock_manager_waiter_lifetime_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_lock_manager_waiter_lifetime_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_lock_manager_waiter_lifetime_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Lock Waiting Queue",
          "subtitle": null,
          "expressions": [
            "sum(max_over_time(\n    tikv_lock_manager_wait_table_status\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30s]\n)) by (type, $additional_groupby) ",
            "sum(max_over_time(\n    tikv_lock_wait_queue_entries_gauge_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30s]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Deadlock detect duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_lock_manager_detect_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_lock_manager_detect_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_lock_manager_detect_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_lock_manager_detect_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_lock_manager_detect_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Detect error",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_lock_manager_error_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Deadlock detector leader",
          "subtitle": null,
          "expressions": [
            "sum(max_over_time(\n    tikv_lock_manager_detector_leader_heartbeat\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30s]\n)) by (instance) "
          ]
        },
        {
          "title": "Total pessimistic locks memory size",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_pessimistic_lock_memory_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "In-memory pessimistic locking result",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_in_memory_pessimistic_locking\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (result, $additional_groupby) "
          ]
        },
        {
          "title": "Pessimistic lock activities",
          "subtitle": "The number of active keys and waiters.",
          "expressions": [
            "sum((\n    tikv_lock_wait_queue_entries_gauge_vec\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Lengths of lock wait queues when transaction enqueues",
          "subtitle": "The length includes the entering transaction itself",
          "expressions": [
            "sum(increase(\n    tikv_lock_wait_queue_length_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "In-memory scan lock read duration",
          "subtitle": "The duration scan in-memory pessimistic locks with read lock",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )",
            "sum(rate(\n    tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Task",
      "panels": [
        {
          "title": "Worker handled tasks",
          "subtitle": "The number of tasks handled by worker",
          "expressions": [
            "sum(rate(\n    tikv_worker_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) "
          ]
        },
        {
          "title": "Worker pending tasks",
          "subtitle": "Current pending and running tasks of worker",
          "expressions": [
            "sum((\n    tikv_worker_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (name, $additional_groupby) "
          ]
        },
        {
          "title": "FuturePool handled tasks",
          "subtitle": "The number of tasks handled by future_pool",
          "expressions": [
            "sum(rate(\n    tikv_futurepool_handled_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, $additional_groupby) "
          ]
        },
        {
          "title": "FuturePool pending tasks",
          "subtitle": "Current pending and running tasks of future_pool",
          "expressions": [
            "sum(avg_over_time(\n    tikv_futurepool_pending_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (name, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "PD",
      "panels": [
        {
          "title": "PD requests",
          "subtitle": "The count of requests that TiKV sends to PD",
          "expressions": [
            "sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "PD request duration (average)",
          "subtitle": "The time consumed by requests that TiKV sends to PD",
          "expressions": [
            "(sum(rate(\n    tikv_pd_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby)  / sum(rate(\n    tikv_pd_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) )"
          ]
        },
        {
          "title": "PD heartbeats",
          "subtitle": "The total number of PD heartbeat messages",
          "expressions": [
            "sum(rate(\n    tikv_pd_heartbeat_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) ",
            "sum((\n    tikv_pd_pending_heartbeat_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "PD validate peers",
          "subtitle": "The total number of peers validated by the PD worker",
          "expressions": [
            "sum(rate(\n    tikv_pd_validate_peer_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "PD reconnection",
          "subtitle": "The count of reconnection between TiKV and PD",
          "expressions": [
            "sum(delta(\n    tikv_pd_reconnect_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "PD forward status",
          "subtitle": "The forward status of PD client",
          "expressions": [
            "((\n    tikv_pd_request_forwarded\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Pending TSO Requests",
          "subtitle": "The number of TSO requests waiting in the queue.",
          "expressions": [
            "sum((\n    tikv_pd_pending_tso_request_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Slow Node Statistics",
      "panels": [
        {
          "title": "Inspected duration per server",
          "subtitle": "The duration that recorded by inspecting messages.",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    store_inspect_disk_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    store_inspect_network_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, target, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Store Slow Score",
          "subtitle": "The slow score of stores",
          "expressions": [
            "sum((\n    store_slow_score\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, type) "
          ]
        }
      ]
    },
    {
      "title": "Snapshot",
      "panels": [
        {
          "title": "Rate snapshot message",
          "subtitle": "The rate of Raft snapshot messages sent",
          "expressions": [
            "sum(delta(\n    tikv_raftstore_raft_sent_message_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [1m]\n)) by (instance) "
          ]
        },
        {
          "title": "Snapshot state count",
          "subtitle": "The number of snapshots in different states",
          "expressions": [
            "sum((\n    tikv_raftstore_snapshot_traffic_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Snapshot generation/apply wait duration $optional_quantile",
          "subtitle": "The time snapshot generation/apply tasks spent waiting to be executed.",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "99% Handle snapshot duration",
          "subtitle": "The time consumed when handling snapshots",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_server_send_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_raftstore_snapshot_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"generate\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "99.99% Snapshot size",
          "subtitle": "The snapshot size (P99.99).9999",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_snapshot_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "99.99% Snapshot KV count",
          "subtitle": "The number of KV within a snapshot in .9999",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_snapshot_kv_count_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Snapshot Actions",
          "subtitle": "Action stats for snapshot generating and applying",
          "expressions": [
            "sum(delta(\n    tikv_raftstore_snapshot_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, status, $additional_groupby) ",
            "sum(delta(\n    tikv_raftstore_clean_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, status, $additional_groupby) ",
            "sum(delta(\n    tikv_server_snapshot_task_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Snapshot transport speed",
          "subtitle": "The speed of sending or receiving snapshot",
          "expressions": [
            "sum(rate(\n    tikv_snapshot_limit_transport_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) ",
            "sum(rate(\n    tikv_snapshot_limit_generate_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Snapshot pending applies",
          "subtitle": "The number of snapshots waiting to be applied",
          "expressions": [
            "((\n    tikv_raftstore_snapshot_pending_applies\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Resolved TS",
      "panels": [
        {
          "title": "Resolved TS Worker CPU",
          "subtitle": "The CPU utilization of resolved ts worker",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Advance ts Worker CPU",
          "subtitle": "The CPU utilization of advance ts worker",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Scan lock Worker CPU",
          "subtitle": "The CPU utilization of scan lock worker",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Max gap of resolved-ts",
          "subtitle": "The gap between resolved ts (the maximum candidate of safe-ts) and current time.",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_resolved_ts_gap_millis\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Min Resolved TS Region",
          "subtitle": "The region that has minimal resolved ts",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_resolved_ts_region\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Max gap of follower safe-ts",
          "subtitle": "The gap between now() and the minimal (non-zero) safe ts for followers",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_follower_safe_ts_gap_millis\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Min Safe TS Follower Region",
          "subtitle": "The region id of the follower that has minimal safe ts",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_follower_safe_ts_region\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Max gap of resolved-ts in region leaders",
          "subtitle": "The gap between resolved ts of leaders and current time",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Min Leader Resolved TS Region",
          "subtitle": "The region that its leader has minimal resolved ts.",
          "expressions": [
            "sum((\n    tikv_resolved_ts_min_leader_resolved_ts_region\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Check leader duration",
          "subtitle": "The time consumed when handle a check leader request",
          "expressions": [
            "sum(increase(\n    tikv_resolved_ts_check_leader_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "CheckLeader request region count $optional_quantile",
          "subtitle": "Bucketed histogram of region count in a check leader request",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_check_leader_request_item_count_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "CheckLeader request size$optional_quantile",
          "subtitle": "Bucketed histogram of the check leader request size",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_check_leader_request_size_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_check_leader_request_item_count_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Fail advance ts count",
          "subtitle": "The count of fail to advance resolved-ts",
          "expressions": [
            "sum(delta(\n    tikv_resolved_ts_fail_advance_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
            "sum(delta(\n    tikv_raftstore_check_stale_peer\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Lock heap size",
          "subtitle": "Total bytes in memory of resolved-ts observe regions's lock heap",
          "expressions": [
            "avg((\n    tikv_resolved_ts_lock_heap_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Initial scan backoff duration",
          "subtitle": "The backoff duration before starting initial scan",
          "expressions": [
            "sum(increase(\n    tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Observe region status",
          "subtitle": "The status of resolved-ts observe regions",
          "expressions": [
            "sum((\n    tikv_resolved_ts_region_resolve_status\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "Pending command size",
          "subtitle": "Total bytes of pending commands in the channel",
          "expressions": [
            "avg((\n    tikv_resolved_ts_channel_penging_cmd_bytes_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Point In Time Restore",
      "panels": [
        {
          "title": "CPU Usage",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "P99 RPC Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_import_apply_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"queue|exec_download\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_import_engine_request_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  "
          ]
        },
        {
          "title": "Import RPC Ops",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_rpc_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n    [$__rate_interval]\n)) by (instance, request) ",
            "sum(rate(\n    tikv_import_rpc_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n    [$__rate_interval]\n)) by (request) "
          ]
        },
        {
          "title": "Import RPC Count",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_import_rpc_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n    \n))  "
          ]
        },
        {
          "title": "Cache Events",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_apply_cache_event\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, instance) "
          ]
        },
        {
          "title": "Overall RPC Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Read File into Memory Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_apply_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"exec_download\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Queuing Time",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_engine_request_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queuing\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Apply Request Throughput",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_apply_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Downloaded File Size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_download_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Apply Batch Size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_apply_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Blocked by Concurrency Time",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_engine_request_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"get_permit\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Apply Request Speed",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_applier_event\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"begin_req\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Cached File in Memory",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_import_apply_cached_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Engine Requests Unfinished",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_applier_event\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"begin_req\"}\n    [$__rate_interval]\n)) by (instance, type) "
          ]
        },
        {
          "title": "Apply Time",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_engine_request_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Raft Store Memory Usage",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_server_mem_trace_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n    \n)) by (instance) "
          ]
        }
      ]
    },
    {
      "title": "Backup & Import",
      "panels": [
        {
          "title": "Backup CPU Utilization",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"b.*k.*w.*k.*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"backup_io\"}\n    [$__rate_interval]\n)) by (instance) ",
            "((\n    tikv_backup_softlimit\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Backup Thread Count",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_backup_thread_pool_size\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Backup Errors",
          "subtitle": null,
          "expressions": [
            "sum(delta(\n    tikv_backup_error_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, error) "
          ]
        },
        {
          "title": "Backup Write CF SST Size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_backup_range_size_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Backup Default CF SST Size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_backup_range_size_bytes_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Backup SST Generation Throughput",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_backup_range_size_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n))  ",
            "sum(rate(\n    tikv_backup_range_size_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, cf) "
          ]
        },
        {
          "title": "Backup Scan SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_backup_range_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Backup Scan SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_backup_range_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Backup Save SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_backup_range_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"save.*\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Backup SST Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.999,(\n    sum(rate(\n    tikv_backup_range_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_backup_range_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, le, $additional_groupby) \n    \n    \n))  ",
            "(sum((\n    tikv_backup_range_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby)  / sum((\n    tikv_backup_range_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (type, $additional_groupby) )"
          ]
        },
        {
          "title": "External Storage Create Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_external_storage_create_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "External Storage Create Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_external_storage_create_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_external_storage_create_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_external_storage_create_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_external_storage_create_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_external_storage_create_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Checksum Request Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_coprocessor_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_coprocessor_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n    [$__rate_interval]\n)) by (req, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_coprocessor_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby)  / sum(rate(\n    tikv_coprocessor_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) )",
            "sum(rate(\n    tikv_coprocessor_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "IO Utilization",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    node_disk_io_time_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, device) "
          ]
        },
        {
          "title": "Import CPU Utilization",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance, tid) > 0",
            "count(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Import Thread Count",
          "subtitle": null,
          "expressions": [
            "count(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Import Errors",
          "subtitle": null,
          "expressions": [
            "sum(delta(\n    tikv_import_error_counter\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, error, instance) "
          ]
        },
        {
          "title": "Import RPC Duration",
          "subtitle": null,
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (request, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (request, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_import_rpc_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (request, $additional_groupby)  / sum(rate(\n    tikv_import_rpc_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (request, $additional_groupby) )",
            "sum(rate(\n    tikv_import_rpc_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (request, $additional_groupby) "
          ]
        },
        {
          "title": "Import RPC Ops",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_rpc_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n    [$__rate_interval]\n)) by (request, $additional_groupby) "
          ]
        },
        {
          "title": "Import RPC Count",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_import_rpc_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Import Write/Download RPC Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"download|write\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Wait Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_download_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queue\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Read SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_download_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"read\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Rewrite SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_download_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"rewrite\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Ingest RPC Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_rpc_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"ingest\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Ingest SST Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_ingest_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"ingest\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Ingest SST Bytes",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tikv_import_ingest_byte_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Import Download SST Throughput",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_download_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) ",
            "sum(rate(\n    tikv_import_download_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n))  "
          ]
        },
        {
          "title": "Import Local Write keys",
          "subtitle": null,
          "expressions": [
            "sum(delta(\n    tikv_import_local_write_keys\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, instance) "
          ]
        },
        {
          "title": "Import Local Write bytes",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_import_local_write_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, instance) "
          ]
        },
        {
          "title": "TTL Expired",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_backup_raw_expired_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_backup_raw_expired_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "cloud request",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_cloud_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (cloud, req) "
          ]
        }
      ]
    },
    {
      "title": "Backup Log",
      "panels": [
        {
          "title": "Endpoint Status",
          "subtitle": null,
          "expressions": [
            "((\n    tikv_log_backup_enabled\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Task Status",
          "subtitle": null,
          "expressions": [
            "min((\n    tikv_log_backup_task_status\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Advancer Owner",
          "subtitle": null,
          "expressions": [
            "tidb_log_backup_advancer_owner > 0"
          ]
        },
        {
          "title": "Average Flush Size",
          "subtitle": "The average flush size of last 30mins.",
          "expressions": [
            "(sum(increase(\n    tikv_log_backup_flush_file_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30m]\n)) by (instance)  / sum(increase(\n    tikv_log_backup_flush_duration_sec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n    [30m]\n)) by (instance) )"
          ]
        },
        {
          "title": "Flushed Files (Last 30m) Per Host",
          "subtitle": "The current total flushed file number of this run.",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_flush_file_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30m]\n)) by (instance) > 0"
          ]
        },
        {
          "title": "Flush Times (Last 30m)",
          "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_flush_duration_sec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n    [30m]\n)) by (instance) "
          ]
        },
        {
          "title": "Total Flushed Size (Last 30m)",
          "subtitle": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_flush_file_size_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30m]\n)) by (instance) "
          ]
        },
        {
          "title": "Flush Files (Last 30m)",
          "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_flush_file_size_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [30m]\n)) by (instance) "
          ]
        },
        {
          "title": "CPU Usage",
          "subtitle": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**",
          "expressions": [
            "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Handle Event Rate",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_log_backup_handle_kv_batch_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Initial Scan Generate Event Throughput",
          "subtitle": "The data rate of initial scanning emitting events.",
          "expressions": [
            "sum(rate(\n    tikv_log_backup_incremental_scan_bytes_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "Abnormal Checkpoint TS Lag",
          "subtitle": null,
          "expressions": [
            "(time() * 1000 - max((\n    tidb_log_backup_last_checkpoint\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (task) / 262144 > 0)"
          ]
        },
        {
          "title": "Memory Of Events",
          "subtitle": "The estimated memory usage by the streaming backup module.",
          "expressions": [
            "sum((\n    tikv_log_backup_heap_memory\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Observed Region Count",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_log_backup_observed_region\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) ",
            "sum((\n    tikv_log_backup_observed_region\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Errors",
          "subtitle": "The errors met when backing up.\n**They are retryable, don't worry.**",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_errors\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, instance) "
          ]
        },
        {
          "title": "Fatal Errors",
          "subtitle": "The errors met when backing up.",
          "expressions": [
            "sum(delta(\n    tikv_log_backup_fatal_errors\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [1m]\n)) by (type, instance) "
          ]
        },
        {
          "title": "Checkpoint TS of Tasks",
          "subtitle": null,
          "expressions": [
            "max((\n    tidb_log_backup_last_checkpoint\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (task) / 262144 > 0",
            "time() * 1000"
          ]
        },
        {
          "title": "Flush Duration",
          "subtitle": "The duration of flushing a batch of file.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_flush_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Initial scanning duration",
          "subtitle": "The duration of scanning the initial data from local DB and transform them into apply events.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_initial_scan_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Convert Raft Event duration",
          "subtitle": "The duration of converting a raft request into a apply event.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_event_handle_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"to_stream_event\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Wait for Lock Duration",
          "subtitle": "The duration of waiting the mutex of the controller.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_event_handle_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"get_router_lock\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Command Batch Size",
          "subtitle": "The number of KV-modify of each raft command observed.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_handle_kv_batch_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Save to Temp File Duration",
          "subtitle": "The total cost of saving an event into temporary file.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_event_handle_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_to_temp_file\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Write to Temp File Duration",
          "subtitle": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_on_event_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"write_to_tempfile\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "System Write Call Duration",
          "subtitle": "The duration of collecting metadata and call the UNIX system call *write* for each event.",
          "expressions": [
            "sum(increase(\n    tikv_log_backup_on_event_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"syscall_write\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Internal Message Type",
          "subtitle": "The internal message type count.",
          "expressions": [
            "sum(rate(\n    tikv_log_backup_internal_actor_acting_duration_sec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (message) "
          ]
        },
        {
          "title": "Internal Message Handling Duration $optional_quantile",
          "subtitle": "The internal handling message duration.",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tikv_log_backup_internal_actor_acting_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (message, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Internal Message Handling Duration (P90)",
          "subtitle": "The internal handling message duration.",
          "expressions": [
            "histogram_quantile(0.9,(\n    sum(rate(\n    tikv_log_backup_internal_actor_acting_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (message, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Initial Scan RocksDB Throughput",
          "subtitle": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.",
          "expressions": [
            "sum(rate(\n    tikv_log_backup_initial_scan_operations\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=~\"read_bytes\"}\n    [$__rate_interval]\n)) by (cf) "
          ]
        },
        {
          "title": "Initial Scan RocksDB Operation",
          "subtitle": "Misc statistics of RocksDB during initial scanning.",
          "expressions": [
            "sum(rate(\n    tikv_log_backup_initial_scan_operations\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op!~\"read_bytes\"}\n    [$__rate_interval]\n)) by (cf, op) > 0"
          ]
        },
        {
          "title": "Initial Scanning Trigger Reason",
          "subtitle": "The reason of triggering initial scanning.",
          "expressions": [
            "sum(rate(\n    tikv_log_backup_initial_scan_reason\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (reason) "
          ]
        },
        {
          "title": "Initial Scanning Task Status",
          "subtitle": "The task status of initial scanning.",
          "expressions": [
            "sum((\n    tikv_log_backup_pending_initial_scan\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (stage) "
          ]
        },
        {
          "title": "Region Checkpoint Key Putting",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_log_backup_metadata_key_operation\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type) "
          ]
        },
        {
          "title": "Request Checkpoint Batch Size",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tidb_log_backup_advancer_batch_size_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"checkpoint\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Tick Duration",
          "subtitle": null,
          "expressions": [
            "sum(increase(\n    tidb_log_backup_advancer_tick_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"tick\"}\n    [$__rate_interval]\n)) by (le) "
          ]
        },
        {
          "title": "Region Checkpoint Failure Reason",
          "subtitle": "The reason of advancer failed to be advanced.",
          "expressions": [
            "sum(rate(\n    tidb_log_backup_region_request_failure\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason!=\"retryable-scan-region\"}\n    [$__rate_interval]\n)) by (reason) "
          ]
        },
        {
          "title": "Request Result",
          "subtitle": "The result of getting region checkpoints.",
          "expressions": [
            "sum(rate(\n    tidb_log_backup_region_request\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (result) "
          ]
        },
        {
          "title": "Tick Duration $optional_quantile",
          "subtitle": "The internal handling message duration.",
          "expressions": [
            "histogram_quantile($optional_quantile,(\n    sum(rate(\n    tidb_log_backup_advancer_tick_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (step, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Tick Duration (P90)",
          "subtitle": "The internal handling message duration.",
          "expressions": [
            "histogram_quantile(0.9,(\n    sum(rate(\n    tidb_log_backup_advancer_tick_duration_sec_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (step, le) \n    \n    \n))  "
          ]
        },
        {
          "title": "Get Region Operation Count",
          "subtitle": "The frequent of getting region level checkpoint.",
          "expressions": [
            "sum(rate(\n    tidb_log_backup_advancer_tick_duration_sec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"get-regions-in-range\"}\n    [$__rate_interval]\n)) by (step, instance) "
          ]
        },
        {
          "title": "Try Advance Trigger Time",
          "subtitle": "The variant of checkpoint group.",
          "expressions": [
            "sum(rate(\n    tidb_log_backup_advancer_tick_duration_sec_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"try-advance\"}\n    [$__rate_interval]\n)) by (step, instance) "
          ]
        }
      ]
    },
    {
      "title": "Threads",
      "panels": [
        {
          "title": "Threads state",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_threads_state\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, state) ",
            "sum((\n    tikv_threads_state\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Threads IO",
          "subtitle": null,
          "expressions": [
            "topk(20,(\n    sum(rate(\n    tikv_threads_io_bytes_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name, io) > 1024\n    \n    \n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "Thread Voluntary Context Switches",
          "subtitle": null,
          "expressions": [
            "topk(20,(\n    max(rate(\n    tikv_thread_voluntary_context_switches\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name) > 100\n    \n    \n))  "
          ]
        },
        {
          "title": "Thread Nonvoluntary Context Switches",
          "subtitle": null,
          "expressions": [
            "topk(20,(\n    max(rate(\n    tikv_thread_nonvoluntary_context_switches\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (name) > 100\n    \n    \n))  "
          ]
        }
      ]
    },
    {
      "title": "Memory",
      "panels": [
        {
          "title": "Allocator Stats",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_allocator_stats\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, type) "
          ]
        },
        {
          "title": "Send Allocated(+) / Release Received(-) Bytes Rate",
          "subtitle": null,
          "expressions": [
            "(sum(rate(\n    tikv_allocator_thread_allocation\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n    [$__rate_interval]\n)) by (thread_name, $additional_groupby)  - sum(rate(\n    tikv_allocator_thread_allocation\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n    [$__rate_interval]\n)) by (thread_name, $additional_groupby) )"
          ]
        },
        {
          "title": "Allocated Bytes Rate per Thread",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_allocator_thread_allocation\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n    [$__rate_interval]\n)) by (thread_name, $additional_groupby) "
          ]
        },
        {
          "title": "Released Bytes Rate per Thread",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_allocator_thread_allocation\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n    [$__rate_interval]\n)) by (thread_name, $additional_groupby) "
          ]
        },
        {
          "title": "Mapped Allocation per Thread",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_allocator_thread_stats\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"mapped\"}\n    \n)) by (thread_name, $additional_groupby) "
          ]
        },
        {
          "title": "Arena Count",
          "subtitle": null,
          "expressions": [
            "sum((\n    tikv_allocator_arena_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Memory Profile Auto Dump Actions",
          "subtitle": "Auto dump action rates grouped by component and action",
          "expressions": [
            "sum(rate(\n    tikv_memory_profile_auto_dump_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr).*\"}\n    [$__rate_interval]\n)) by (component, action) "
          ]
        }
      ]
    },
    {
      "title": "Resource Control",
      "panels": [
        {
          "title": "Background Task Total Wait Duration",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_resource_control_background_task_wait_duration\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance, resource_group) "
          ]
        },
        {
          "title": "Priority Quota Limit",
          "subtitle": "The memory usage of the resource control module.",
          "expressions": [
            "sum((\n    tikv_resource_control_priority_quota_limit\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance, priority) "
          ]
        }
      ]
    },
    {
      "title": "Status Server",
      "panels": [
        {
          "title": "Status API Request Duration",
          "subtitle": "The 99 quantile durtion of status server API requests",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_status_server_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_status_server_request_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_status_server_request_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, $additional_groupby)  / sum(rate(\n    tikv_status_server_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, $additional_groupby) )",
            "sum(rate(\n    tikv_status_server_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, $additional_groupby) "
          ]
        },
        {
          "title": "Status API Request (op/s)",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_status_server_request_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (path, $additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "Encryption",
      "panels": [
        {
          "title": "Encryption data keys",
          "subtitle": "Total number of encryption data keys in use",
          "expressions": [
            "sum((\n    tikv_encryption_data_key_storage_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Encrypted files",
          "subtitle": "Number of files being encrypted",
          "expressions": [
            "sum((\n    tikv_encryption_file_num\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n)) by (instance) "
          ]
        },
        {
          "title": "Encryption initialized",
          "subtitle": "Flag to indicate if encryption is initialized",
          "expressions": [
            "((\n    tikv_encryption_is_initialized\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Encryption meta files size",
          "subtitle": "Total size of encryption meta files",
          "expressions": [
            "((\n    tikv_encryption_meta_file_size_bytes\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    \n))  "
          ]
        },
        {
          "title": "Encrypt/decrypt data nanos",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"encrypt_data_nanos\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) ",
            "sum(rate(\n    tikv_coprocessor_rocksdb_perf\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"decrypt_data_nanos\"}\n    [$__rate_interval]\n)) by (req, $additional_groupby) "
          ]
        },
        {
          "title": "Read/write encryption meta duration",
          "subtitle": "Writing or reading file duration (second)",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_encryption_write_read_file_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_encryption_write_read_file_duration_seconds_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_encryption_write_read_file_duration_seconds_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_encryption_write_read_file_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_encryption_write_read_file_duration_seconds_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        }
      ]
    },
    {
      "title": "TTL",
      "panels": [
        {
          "title": "TTL expire count",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_ttl_expire_kv_count_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "TTL expire size",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_ttl_expire_kv_size_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance) "
          ]
        },
        {
          "title": "TTL check progress",
          "subtitle": null,
          "expressions": [
            "(sum(rate(\n    tikv_ttl_checker_processed_regions\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (instance)  / sum(rate(\n    tikv_raftstore_region_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n    [$__rate_interval]\n)) by (instance) )"
          ]
        },
        {
          "title": "TTL checker actions",
          "subtitle": null,
          "expressions": [
            "sum(rate(\n    tikv_ttl_checker_actions\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (type, $additional_groupby) "
          ]
        },
        {
          "title": "TTL checker compact duration",
          "subtitle": "The time consumed when executing GC tasks",
          "expressions": [
            "histogram_quantile(0.9999,(\n    sum(rate(\n    tikv_ttl_checker_compact_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "histogram_quantile(0.99,(\n    sum(rate(\n    tikv_ttl_checker_compact_duration_bucket\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by (le, $additional_groupby) \n    \n    \n))  ",
            "(sum(rate(\n    tikv_ttl_checker_compact_duration_sum\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby)  / sum(rate(\n    tikv_ttl_checker_compact_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) )",
            "sum(rate(\n    tikv_ttl_checker_compact_duration_count\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n    [$__rate_interval]\n)) by ($additional_groupby) "
          ]
        },
        {
          "title": "TTL checker poll interval",
          "subtitle": null,
          "expressions": [
            "max((\n    tikv_ttl_checker_poll_interval\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"tikv_gc_run_interval\"}\n    \n))  "
          ]
        }
      ]
    }
  ]
}