{
  "panels": [
    {
      "main_title": "Server",
      "sub_title": "Store size",
      "expressions": [
        "sum(tiflash_system_current_metric_StoreSizeUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"\"}) by (instance)",
        "sum(tiflash_system_current_metric_StoreSizeUsedRemote{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "Available size",
      "expressions": [
        "sum(tiflash_system_current_metric_StoreSizeAvailable{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "Capacity size",
      "expressions": [
        "sum(tiflash_system_current_metric_StoreSizeCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "Uptime",
      "expressions": [
        "tiflash_system_asynchronous_metric_Uptime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "Region",
      "expressions": [
        "sum(tiflash_proxy_tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"region\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "CPU Usage",
      "expressions": [
        "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m])",
        "sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "Memory",
      "expressions": [
        "sum(tiflash_system_asynchronous_metric_jemalloc_retained{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_mapped{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_resident{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_allocated{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_active{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_jemalloc_metadata{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})",
        "sum(tiflash_system_asynchronous_metric_mimalloc_current_rss{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})",
        "sum(tiflash_system_asynchronous_metric_mimalloc_current_commit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})",
        "sum(tiflash_system_asynchronous_metric_mmap_alive{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"})",
        "tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}",
        "sum(tiflash_system_current_metric_MemoryCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Server",
      "sub_title": "IO Throughput",
      "expressions": [
        "sum by (instance) (irate(tiflash_proxy_threads_io_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}[2m]))"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "SST Import Service",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"sst_importer.*\"}[2m]))"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "SST Apply",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_low_.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Region Task",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_task.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Region Worker",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"region_worker.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Raft Store",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"raftstore_.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Apply Worker",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"apply_.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Storage Background (Small Tasks)",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_\\\\d+\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Storage Background (Large Tasks)",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"bg_block_\\\\d+\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Manual Compaction",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"m_compact_pool\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "GRPC Async Server",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"async_poller.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "GRPC Async Client",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "FAP builder",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"fap_builder.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"GRPCComp.*\"}) < sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance) or sum(tiflash_system_current_metric_LogicalCPUCores{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Snapshot Sender",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"snap_sender.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Segment Scheduler",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"segment_sched.*\"})"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Local Index Pool",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexPool*\"})",
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"LocalIndexSched*\"}[2m]))"
      ]
    },
    {
      "main_title": "Threads CPU",
      "sub_title": "Segment Reader",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"SegmentReader.*\"})"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Request QPS",
      "expressions": [
        "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Executor QPS",
      "expressions": [
        "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Request Duration",
      "expressions": [
        "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Error QPS",
      "expressions": [
        "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (reason)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Request Handle Duration",
      "expressions": [
        "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Response Bytes/Seconds",
      "expressions": [
        "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Cop task memory usage",
      "expressions": [
        "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Exchange Bytes/Seconds",
      "expressions": [
        "sum(rate(tiflash_exchange_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Threads of Rpc",
      "expressions": [
        "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type!~\".*max\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Handling Request Number",
      "expressions": [
        "sum(tiflash_coprocessor_handling_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Threads",
      "expressions": [
        "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\".*max\", type!~\"rpc.*\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Max Threads of Rpc",
      "expressions": [
        "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"rpc.*\", type=~\".*max\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "MPP Query count",
      "expressions": [
        "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Max Threads",
      "expressions": [
        "max(tiflash_thread_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*max\", type!~\"rpc.*\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Time of the Longest Live MPP Task",
      "expressions": [
        "tiflash_mpp_task_monitor{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Coprocessor",
      "sub_title": "Data size in send and receive queue",
      "expressions": [
        "tiflash_exchange_queueing_data_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Min TSO",
      "expressions": [
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"min_tso\"}) by (instance, resource_group)"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Estimated Thread Usage and Limit",
      "expressions": [
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_soft_limit\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"estimated_thread_usage\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"thread_hard_limit\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"global_estimated_thread_usage\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"group_entry_count\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Active and Waiting Queries Count",
      "expressions": [
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_queries_count\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_queries_count\"}) by (instance, type, resource_group)"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Active and Waiting Tasks Count",
      "expressions": [
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"waiting_tasks_count\"}) by (instance, type, resource_group)",
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"active_tasks_count\"}) by (instance, type, resource_group)"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Hard Limit Exceeded Count",
      "expressions": [
        "max(tiflash_task_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"hard_limit_exceeded_count\"}) by (instance, type, resource_group)"
      ]
    },
    {
      "main_title": "Task Scheduler",
      "sub_title": "Task Waiting Duration",
      "expressions": [
        "histogram_quantile(0.80, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))",
        "histogram_quantile(0.90, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))",
        "histogram_quantile(1.00, max(rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance,le,resource_group))"
      ]
    },
    {
      "main_title": "DDL",
      "sub_title": "Schema Internal DDL OPM",
      "expressions": [
        "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)",
        "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))",
        "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type,instance)",
        "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)"
      ]
    },
    {
      "main_title": "DDL",
      "sub_title": "Schema Apply OPM",
      "expressions": [
        "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)"
      ]
    },
    {
      "main_title": "DDL",
      "sub_title": "Schema Apply Duration",
      "expressions": [
        "histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", keyspace!=\"\"}[2m])) by (le, type, keyspace))",
        "sum(tiflash_sync_schema_applying{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"$type\"}) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Write Command OPS",
      "expressions": [
        "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)",
        "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Write Amplification",
      "expressions": [
        "sum by (instance) (\ntiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"} +\ntiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}\n)\n/\nsum by (instance) (\ntiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}\n)",
        "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)",
        "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[10m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[10m])\n)",
        "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[30m])\n)\n/\nsum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[30m])\n)",
        "sum by (instance) (\nrate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) +\nrate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])\n)",
        "sum by (instance) (\nrate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[5m])\n)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Read Tasks OPS",
      "expressions": [
        "sum(rate(tiflash_storage_read_tasks_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "SubTasks Write Throughput (bytes)",
      "expressions": [
        "sum(rate(tiflash_storage_subtask_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "SubTasks Write Throughput (rows)",
      "expressions": [
        "sum(rate(tiflash_storage_subtask_throughput_rows{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Small Internal Tasks OPS",
      "expressions": [
        "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Small Internal Tasks Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Large Internal Tasks OPS",
      "expressions": [
        "sum(rate(tiflash_storage_subtask_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Large Internal Tasks Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le,type) / 1000000000)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Opened File Count",
      "expressions": [
        "tiflash_proxy_process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", component=\"tiflash\"}",
        "sum(tiflash_system_current_metric_OpenFileForWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_OpenFileForRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_OpenFileForReadWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "File Open OPS",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_FileOpen{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)",
        "sum(rate(tiflash_system_profile_event_FileOpenFailed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "FSync Status",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_FileFSync{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)",
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Disk Write OPS",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_PSMWritePages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Disk Read OPS",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_PSMReadIOCalls{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_PSMReadPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Write flow",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Read flow",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))",
        "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Current Data Management Tasks",
      "expressions": [
        "avg(tiflash_system_current_metric_DT_DeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_DT_SegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_DT_SegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "DeltaIndexError",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_DTDeltaIndexError{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "I/O Limiter Throughput",
      "expressions": [
        "sum(rate(tiflash_storage_io_limiter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "I/O Limiter Pending Rate and Duration",
      "expressions": [
        "sum(rate(tiflash_storage_io_limiter_pending_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Data Sharing",
      "expressions": [
        "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"add_cache_stale|add_cache_succ|add_cache_total_bytes_limit|add_cache_reach_count_limit\"}[2m])) by (type)",
        "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))",
        "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))",
        "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_hit|get_cache_copy\"}[2m]))/sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"get_cache_miss|get_cache_hit|get_cache_part|get_cache_copy\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "I/O Limiter Current Pending Count",
      "expressions": [
        "avg(tiflash_system_current_metric_RateLimiterPendingWriteRequest{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_IOLimiterPendingBgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_IOLimiterPendingFgWriteReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_IOLimiterPendingBgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "avg(tiflash_system_current_metric_IOLimiterPendingFgReadReq{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Effectiveness of Mark Cache",
      "expressions": [
        "max(tiflash_system_profile_event_MarkCacheMisses{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "max(tiflash_system_profile_event_MarkCacheHits{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Read Thread Scheduling",
      "expressions": [
        "sum(rate(tiflash_storage_read_thread_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Read Thread Internal Duration",
      "expressions": [
        "histogram_quantile(0.999, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Mark Cache and Minmax Index Cache Memory Usage",
      "expressions": [
        "tiflash_system_asynchronous_metric_MarkCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_MinMaxIndexFiles{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_RNDeltaIndexCacheBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Read Snapshots",
      "expressions": [
        "tiflash_system_current_metric_DT_SegmentReadTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_PSMVCCSnapshotsList{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_PSMVCCNumSnapshots{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfRead{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfReadRaw{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfDeltaMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfDeltaCompact{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfSegmentMerge{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfSegmentSplit{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_current_metric_DT_SnapshotOfPlaceIndex{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_MaxDTStableOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_MaxDTMetaOldestSnapshotLifetime{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Memory by class",
      "expressions": [
        "tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Memory by thread",
      "expressions": [
        "tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"alloc_.*\"}",
        "-tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dealloc_.*\"}"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Schema of Column File",
      "expressions": [
        "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"current_size\"}) by (instance)",
        "sum(rate(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit_count\"}[2m])) by (instance)",
        "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"still_used_when_evict\"}) by (instance)",
        "max(tiflash_shared_block_schemas{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"miss_count\"}) by (instance)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Compression Algorithm Count",
      "expressions": [
        "sum(rate(tiflash_storage_pack_compression_algorithm_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Storage",
      "sub_title": "Compression Ratio",
      "expressions": [
        "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lz4_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lz4_compressed_bytes\"}[2m]))",
        "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lightweight_uncompressed_bytes\"}[2m]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"lightweight_compressed_bytes\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage Write Stall",
      "sub_title": "Write Stall Duration",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type, instance))",
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type, instance) / 1000000000)"
      ]
    },
    {
      "main_title": "Storage Write Stall",
      "sub_title": "Write & Delta Management Throughput",
      "expressions": [
        "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"}[2m]))",
        "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"}[2m]))"
      ]
    },
    {
      "main_title": "Storage Write Stall",
      "sub_title": "Write & Delta Management Total",
      "expressions": [
        "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write|ingest\"})",
        "sum(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type!~\"write|ingest\"})"
      ]
    },
    {
      "main_title": "Storage Write Stall",
      "sub_title": "Write Throughput By Instance",
      "expressions": [
        "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write\"}[2m])) by (instance)",
        "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Storage Write Stall",
      "sub_title": "Write Command OPS By Instance",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_DMWriteBlock{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)",
        "sum(increase(tiflash_storage_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, type)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PageStorage Disk Usage",
      "expressions": [
        "tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "sum(tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"})) by (instance)",
        "tiflash_system_asynchronous_metric_LogDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PageStorage File Num",
      "expressions": [
        "sum(tiflash_system_asynchronous_metric_BlobFileNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_asynchronous_metric_LogNums{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PageStorage WriteBatch Size",
      "expressions": [
        "sum(delta(tiflash_storage_page_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"v3\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "Page write Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.999, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.99, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "Page GC Tasks OPM",
      "expressions": [
        "sum(increase(tiflash_storage_page_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "Page GC Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "Numer of Pages",
      "expressions": [
        "tiflash_system_asynchronous_metric_PagesInMem{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}",
        "tiflash_system_asynchronous_metric_VersionedEntries{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "Number of Tables",
      "expressions": [
        "sum(tiflash_system_current_metric_StoragePoolV2Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_StoragePoolV3Only{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_StoragePoolMixMode{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_StoragePoolUniPS{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PageStorage stored bytes by type",
      "expressions": [
        "sum(tiflash_storage_page_data_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PageStorage Pending Writers Num",
      "expressions": [
        "sum(tiflash_system_current_metric_PSPendingWriterNum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "PageStorage",
      "sub_title": "PS Command OPS By Instance",
      "expressions": [
        "sum(rate(tiflash_storage_page_command_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Stale Read OPS",
      "expressions": [
        "sum(rate(tiflash_stale_read_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Read Index OPS",
      "expressions": [
        "sum(rate(tiflash_raft_read_index_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Learner Read Failures",
      "expressions": [
        "sum(rate(tiflash_raft_learner_read_failures_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Read Index Events",
      "expressions": [
        "sum(rate(tiflash_raft_read_index_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Wait Index Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Batch Read Index Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Apply Raft write logs Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) ",
        "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) ",
        "sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"flush_region\"}[2m]))",
        "sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", cluster_id=~\".*$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m]) ) "
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Heavy Raft Apply Duration",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Region write Duration (decode)",
      "expressions": [
        "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"decode\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Region write Duration (write blocks)",
      "expressions": [
        "sum(delta(tiflash_raft_write_data_to_storage_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Apply Raft write logs Duration [Heatmap]",
      "expressions": [
        "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"write\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Apply Raft admin logs Duration [Heatmap]",
      "expressions": [
        "sum(delta(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"admin\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Events QPS",
      "expressions": [
        "sum(rate(tiflash_raft_raft_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Frequent Events QPS",
      "expressions": [
        "sum(rate(tiflash_raft_raft_frequent_events_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Log Gap Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"applied_index\"}[2m])) by (le, type)",
        "sum(delta(tiflash_raft_raft_log_gap_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"compact_index\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Entry Batch Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_entry_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"normal\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Region Size (by event) Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"unflushed\"}[2m])) by (le, type)",
        "sum(delta(tiflash_raft_region_flush_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"flushed\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Ongoing raft snapshot",
      "expressions": [
        "sum(rate(tiflash_raft_ongoing_snapshot_total_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Write Committed Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"write_committed\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Uncommitted Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"snapshot_uncommitted\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Ingest Uncommitted Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest_uncommitted\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Big Write To Region Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_write_flow_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"big_write_to_region\"}[2m])) by (le, type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Eager GC OPS",
      "expressions": [
        "sum(rate(tiflash_raft_eager_gc_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[$__rate_interval])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft Eager GC Duration",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.95, sum(rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "sum(rate(tiflash_raft_eager_gc_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type) / sum(rate(tiflash_raft_eager_gc_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Applying snapshots Count",
      "expressions": [
        "sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_RaftNumPrehandlingSubTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_RaftNumParallelPrehandlingTasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Predecode Duration",
      "expressions": [
        "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Predecode SST to DT Duration",
      "expressions": [
        "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_predecode_sst2dt\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "KVStore memory",
      "expressions": [
        "sum(tiflash_system_current_metric_MemoryTrackingKVStore{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Memory by thread",
      "expressions": [
        "tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"alloc_.*\"}",
        "-tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dealloc_.*\"}"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Prehandle Throughput Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_command_throughput_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"prehandle_snapshot\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Size Heatmap",
      "expressions": [
        "sum(delta(tiflash_raft_snapshot_total_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"approx_raft_snapshot\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Keys flow",
      "expressions": [
        "sum(rate(tiflash_raft_process_keys{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Raft throughput",
      "expressions": [
        "sum(rate(tiflash_raft_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Snapshot Flush Duration",
      "expressions": [
        "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot_flush\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Ingest SST Duration",
      "expressions": [
        "sum(delta(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"ingest_sst\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Upstream Latency [Heatmap]",
      "expressions": [
        "sum(delta(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Raft",
      "sub_title": "Upstream Latency",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "histogram_quantile(0.95, sum(rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le))",
        "sum(rate(tiflash_raft_upstream_latency_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / sum(rate(tiflash_raft_upstream_latency_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) "
      ]
    },
    {
      "main_title": "Rough Set Filter Rate Histogram",
      "sub_title": "Rough Set Filter Rate",
      "expressions": [
        "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (instance)",
        "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[5m]))) by (instance)",
        "sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)",
        "sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)",
        "sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Rough Set Filter Rate Histogram",
      "sub_title": "Rough Set Filter Rate Histogram",
      "expressions": [
        "sum(delta(tiflash_storage_rough_set_filter_rate_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le)"
      ]
    },
    {
      "main_title": "Vector Search",
      "sub_title": "In-Memory Vector Index Instances",
      "expressions": [
        "sum by (type, instance) (\n    tiflash_vector_index_active_instances{\n        k8s_cluster=\"$k8s_cluster\", \n        tidb_cluster_id=\"$tidb_cluster\", \n        instance=~\"$instance\"\n    }\n)"
      ]
    },
    {
      "main_title": "Vector Search",
      "sub_title": "Vector Index Estimated Memory Usage",
      "expressions": [
        "tiflash_vector_index_memory_usage{\n    k8s_cluster=\"$k8s_cluster\", \n    tidb_cluster_id=\"$tidb_cluster\", \n    instance=~\"$instance\"\n}",
        "tiflash_process_rss_by_type_bytes{\n    k8s_cluster=\"$k8s_cluster\", \n    tidb_cluster_id=\"$tidb_cluster\", \n    instance=~\"$instance\",\n    type=\"file\"\n}"
      ]
    },
    {
      "main_title": "Vector Search",
      "sub_title": "99.9% Vector Search Duration (Per Request)",
      "expressions": [
        "histogram_quantile(\n    0.999, \n    sum(rate(\n        tiflash_vector_index_duration_bucket{\n            k8s_cluster=\"$k8s_cluster\", \n            tidb_cluster_id=\"$tidb_cluster\", \n            instance=~\"$instance\",\n            type!=\"build\"\n        }\n        [$__rate_interval]\n    )) by (le, type)\n)"
      ]
    },
    {
      "main_title": "Vector Search",
      "sub_title": "99.9% Vector Index Build Duration (Per DMFile Column)",
      "expressions": [
        "histogram_quantile(\n    0.999, \n    sum(rate(\n        tiflash_vector_index_duration_bucket{\n            k8s_cluster=\"$k8s_cluster\", \n            tidb_cluster_id=\"$tidb_cluster\", \n            instance=~\"$instance\",\n            type=\"build\"\n        }\n        [$__rate_interval]\n    )) by (le, type)\n)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Read Duration Breakdown",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Checkpoint Upload Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.90, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Checkpoint Upload flow",
      "expressions": [
        "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"incremental\"}[2m]))",
        "sum(rate(tiflash_storage_checkpoint_flow{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"compaction\"}[2m]))"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Checkpoint Upload keys speed by type (all)",
      "expressions": [
        "sum(rate(tiflash_storage_checkpoint_keys_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Checkpoint Upload flow by type (incremental+compaction)",
      "expressions": [
        "sum(rate(tiflash_storage_checkpoint_flow_by_types{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote File Num",
      "expressions": [
        "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"num_files\"}) by (instance)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote Store Usage",
      "expressions": [
        "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"}) by (instance)",
        "sum(tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) by (instance)",
        "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"total_size\"})) by (instance)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote Object Lock Request QPS",
      "expressions": [
        "sum(rate(tiflash_disaggregated_object_lock_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote Object Lock Duration",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote GC Duration Breakdown",
      "expressions": [
        "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Remote GC Status",
      "expressions": [
        "sum(tiflash_storage_s3_gc_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance,type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Request Unit",
      "expressions": [
        "sum(rate(tiflash_storage_sync_replica_ru[1m])) by (keyspace_id)",
        "sum(increase(tiflash_storage_sync_replica_ru[24h])) by (keyspace_id)",
        "sum(rate(tiflash_compute_request_unit[1m])) by (cluster_id)",
        "sum(increase(tiflash_compute_request_unit[24h])) by (cluster_id)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "Memory Usage of Storage Tasks",
      "expressions": [
        "sum(tiflash_system_current_metric_MemoryTrackingQueryStorageTask{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_MemoryTrackingFetchPages{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_DT_DeltaIndexCacheSize{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_MemoryTrackingSharedColumnData{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "DeltaIndexCache",
      "expressions": [
        "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type, instance)",
        "sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"hit\"}[2m])) by (instance) /sum(rate(tiflash_storage_delta_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (instance)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "FAP state",
      "expressions": [
        "sum(rate(tiflash_fap_task_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "FAP result",
      "expressions": [
        "sum(rate(tiflash_fap_task_result{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "FAP no match reason",
      "expressions": [
        "sum(rate(tiflash_fap_nomatch_reason{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\n"
      ]
    },
    {
      "main_title": "Disaggregated",
      "sub_title": "FAP time by stage",
      "expressions": [
        "histogram_quantile(0.999, sum(round(1000000000*rate(tiflash_fap_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "S3 Bytes",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_S3WriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3ReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3WriteDMFileBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "S3 OPS",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_S3PutObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3GetObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3HeadObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3ListObjects{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3DeleteObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3CopyObject{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3CreateMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3UploadPart{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3CompleteMultipartUpload{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3PutDMFile{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "S3 Retry OPS",
      "expressions": [
        "sum(rate(tiflash_system_profile_event_S3GetObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3PutObjectRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_system_profile_event_S3PutDMFileRetry{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "S3 Request Duration",
      "expressions": [
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)",
        "histogram_quantile(0.99, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.90, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))",
        "histogram_quantile(0.80, sum(rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "Remote Cache Operations",
      "expressions": [
        "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)",
        "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dtfile_hit|dtfile_miss\"}[2m]))",
        "sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit\"}[2m]))/sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\"page_hit|page_miss\"}[2m]))"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "Remote Cache Flow",
      "expressions": [
        "sum(rate(tiflash_storage_remote_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)"
      ]
    },
    {
      "main_title": "S3",
      "sub_title": "Remote Cache Usage",
      "expressions": [
        "sum(tiflash_system_current_metric_DTFileCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_DTFileCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_PageCacheCapacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)",
        "sum(tiflash_system_current_metric_PageCacheUsed{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance)"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Task Thread Pool Size",
      "expressions": [
        "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_task_thread_pool_size\"}) by (instance, type)"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Task Count",
      "expressions": [
        "max(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (instance, type)",
        "sum(tiflash_pipeline_scheduler{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=~\".*_tasks_count\"}) by (type)"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Task Status Change OPS",
      "expressions": [
        "sum(rate(tiflash_pipeline_task_change_to_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (type)\r\n"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Task Duration",
      "expressions": [
        "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r",
        "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r",
        "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r",
        "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_execute\"}[2m]))\r\n\r\n\r\n\r",
        "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu_queue\"}[2m]))\r\n\r",
        "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_execute\"}[2m]))\r\n\r",
        "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io_queue\"}[2m]))\r",
        "sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"await\"}[2m]))\r"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Task Max Execute Time Per Round",
      "expressions": [
        "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r\n\r",
        "histogram_quantile(0.95, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r",
        "histogram_quantile(0.99, sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m])) by (le, type))\r",
        "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\"}[2m]))) by (le, type) / 1000000000)\r",
        "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"cpu\"}[2m]))\r\n\r\n\r\n\r",
        "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", instance=~\"$instance\", type=\"io\"}[2m]))\r"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Threads CPU of CPU Task Thread Pool",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"cpu_pool\"})"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Threads CPU of IO Task Thread Pool",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"io_pool\"})"
      ]
    },
    {
      "main_title": "Pipeline Model",
      "sub_title": "Threads CPU of Wait Reactor",
      "expressions": [
        "sum by (instance) (rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"}[2m]))",
        "count by (instance) (tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", name=~\"WaitReactor\"})"
      ]
    },
    {
      "main_title": "TiFlash Resource Control",
      "sub_title": "TiFlash Resource Group",
      "expressions": [
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"remaining_tokens\", instance=~\"$instance\"}) by (instance,resource_group)",
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"avg_speed\", instance=~\"$instance\"}) by (instance,resource_group)",
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"total_consumption\", instance=~\"$instance\"}) by (instance,resource_group)",
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_fill_rate\", instance=~\"$instance\"}) by (instance,resource_group)",
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"bucket_capacity\", instance=~\"$instance\"}) by (instance,resource_group)",
        "max(tiflash_resource_group{k8s_cluster=\"$k8s_cluster\", tidb_cluster_id=\"$tidb_cluster\", type=\"fetch_tokens_from_gac_count\", instance=~\"$instance\"}) by (instance,resource_group)"
      ]
    }
  ]
}