- name: PD_memory_usage_avg:total
  unit: "Bytes"
  expr: 'label_replace(avg(process_resident_memory_bytes{{label_cluster_id},{label_component}="pd"}), "name","pd_memory_avg","","")'
  component: PD

- name: PD_memory_usage_delta:total
  unit: "Bytes"
  expr: 'label_replace((max(process_resident_memory_bytes{{label_cluster_id},{label_component}="pd"})-min(process_resident_memory_bytes{{label_cluster_id},{label_component}="pd"})), "name","pd_memory_delta","","")'
  component: PD

- name: PD_memory_usage_max:total
  unit: "Bytes"
  expr: 'label_replace(max(process_resident_memory_bytes{{label_cluster_id},{label_component}="pd"}), "name","pd_memory_max","","")'
  component: PD

- name: PD_leader:by_instance
  unit: "%"
  expr: 'label_replace(topk(1,max(irate(process_cpu_seconds_total{{label_cluster_id},{label_component}=~".*pd.*"}[2m]))by(instance))*100,"name","$1","instance","(.*)")'
  rawdata: true
  component: PD

- name: Component:total
  unit: "1"
  expr: 'label_replace(sum(count(process_cpu_seconds_total{{label_cluster_id}, {label_component}=~"pd|tidb|tikv"} > 0) by (component)) + count(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"} > 0),"name","component_total","","")'
  component: TiDB

- name: PD_count:total
  unit: "1"
  expr: 'label_replace(count(process_cpu_seconds_total{{label_cluster_id}, {label_component}="pd"} > 0) by (component),"name","pd_total","","")'
  component: TiDB

- name: TiDB_count:total
  unit: "1"
  expr: 'label_replace(count(process_cpu_seconds_total{{label_cluster_id}, {label_component}="tidb"} > 0) by (component),"name","tidb_total","","")'
  component: TiDB

- name: TiKV_count:total
  unit: "1"
  expr: 'label_replace(count(process_cpu_seconds_total{{label_cluster_id}, {label_component}="tikv"} > 0) by (component),"name","tikv_count","","")'
  component: TiDB

- name: TiFlash_count:total
  unit: "1"
  expr: 'label_replace(count(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"} > 0),"name","tiflash_count","","")'
  component: TiDB
  
#- name: DDL_owner_name
- name: DDL_owner:by_instance
  unit: "1"
#  expr: 'label_replace(sum(increase(tidb_ddl_worker_operation_total{{label_cluster_id},type=~"owner.*"}[2m])) by (instance),"name","DDL_owner_name","","")'
  expr: 'label_replace(sum(increase(tidb_ddl_worker_operation_total{{label_cluster_id},type=~"owner.*"}[2m])) by (instance),"name","$1","instance","(.*)")'
  rawdata: true
  component: TiDB

- name: TiKV_storage_size:total
  unit: "Bytes"
  expr: 'label_replace(sum(tikv_engine_size_bytes{{label_cluster_id}}),"name","total","","")'
  component: TiKV
  
- name: TiKV_storage_capacity:total
  unit: "Bytes"
  expr: 'label_replace(sum(tikv_store_size_bytes{{label_cluster_id}, type="capacity"}) ,"name","total","","")'
  component: TiKV
  
- name: TiFlash_storage_size:total
  unit: "Bytes"
  expr: 'label_replace(sum(tiflash_system_current_metric_StoreSizeUsed{{label_cluster_id}}),"name","total","","")'
  component: TiFlash

- name: TiFlash_storage_capacity:total
  unit: "Bytes"
  expr: 'label_replace(sum(tiflash_system_current_metric_StoreSizeCapacity{{label_cluster_id}}),"name","total","","")'
  component: TiFlash

- name: KV_Request_OPS_Client:by_type
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_tikvclient_request_seconds_count{ {label_cluster_id}}[2m])) by (type),"name","$1","type","(.*)")'
  component: TiDB
 
- name: KV_Request_OPS_tso_command:total
  unit: "1"
  expr: 'label_replace(sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{ {label_cluster_id}, type="tso"}[2m])),"name","total","","")'
  component: TiDB
  
- name: expensive_executor_OPS:total
  unit: "1"
  expr: 'label_replace(sum(sum(rate(tidb_executor_expensive_total{{label_cluster_id}}[2m])) by (type)),"name","total","","")'
  component: TiDB

#- name: SQL_execute_time:top1_by_txnmode
- name: SQL_execute_count_top1:by_orthogonality
  unit: "1"
  expr: 'label_replace(label_replace(topk(1,sum(rate(tidb_session_transaction_duration_seconds_count{{label_cluster_id}}[2m])) by (type, txn_mode)),"name1","$1","type","(.*)"),"name2","$1","txn_mode","(.*)")'
  component: TiDB

#- name: 2PC_dominated:top1_by_type
- name: 2PC_dominated_top1:by_type
  unit: "1"
  expr: 'label_replace(topk(1,sum(rate(tidb_tikvclient_commit_txn_counter{{label_cluster_id}}[2m])) by (type)),"name","$1","type","(.*)")'
  component: TiKV
  
#- name: 1PC_dominated:top1_by_type
- name: 1PC_dominated_top1:by_type
  unit: "s"
  expr: 'label_replace(topk(1,sum(rate(tidb_tikvclient_one_pc_txn_counter{{label_cluster_id}}[2m])) by (type)),"name","$1","type","(.*)")'
  component: TiKV
  
#- name: Async_Commmit_Dominated:top1_by_type
- name: Async_Commmit_Dominated_top1:by_type
  unit: "1"
  expr: 'label_replace(topk(1,sum(rate(tidb_tikvclient_async_commit_txn_counter{{label_cluster_id}}[2m])) by (type)),"name","$1","type","(.*)")'
  component: TiKV
  
- name: Txn_write_size_bytes_avg:total
  unit: "Bytes"
  expr: 'label_replace(avg(rate(tidb_tikvclient_txn_write_size_bytes_sum{{label_cluster_id},job=~".*tidb"}[2m])),"name","total","","")'
  component: TiDB

#- name: Txn_write_size_bytes_delta
- name: Txn_write_size_bytes_delta:total
  unit: "Bytes"
  expr: 'label_replace((max(rate(tidb_tikvclient_txn_write_size_bytes_sum{{label_cluster_id}}[2m])) - min(rate(tidb_tikvclient_txn_write_size_bytes_sum{{label_cluster_id}}[2m]))),"name","delta","","")'
  component: TiDB

- name: Pessimistic_retry_count:total
  unit: "1"
  expr: 'label_replace(sum(increase(tidb_session_statement_pessimistic_retry_count_bucket{{label_cluster_id}}[2m]))by(le),"name","total","","")'
  component: TiDB

#- name: Scheduler_flow:average
- name: Scheduler_flow_average:total
  unit: "Bytes"
  expr: 'label_replace( avg(tikv_scheduler_write_flow{{label_cluster_id}}),"name","total","","")'
  component: TiKV

- name: P99_commit_token_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_tikvclient_batch_executor_token_wait_duration_bucket{{label_cluster_id}}[2m])) by (le)),"name","total","","")'
  component: TiDB

#- name: Connnection_count:detla
- name: Connnection_count_detla:total
  unit: "1"
  expr: 'label_replace(((max(tidb_server_connections{{label_cluster_id}}) - min(tidb_server_connections{{label_cluster_id}}))),"name","connection_delta","","")'
  component: TiDB

- name: Prepare_Statement_count:total
  unit: "1"
  expr: 'label_replace(sum(tidb_server_prepared_stmts{{label_cluster_id}}),"name","total","","")'
  component: TiDB

#- name: Gc_worker_thread_cpu:delta
- name: Gc_worker_thread_cpu_delta:total
  unit: "%"
  expr: 'label_replace((max(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id}, name=~"gc_worker.*"}[2m])) by (instance)) - min(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"gc_worker.*"}[2m])) by (instance)))*100,"name","tikv_gc_delta","","")'
  component: TiKV

#- name: Gc_Progress_average
- name: Gc_Progress_average:total
  unit: "%"
  expr: 'label_replace(avg(sum(tikv_gcworker_autogc_processed_regions{{label_cluster_id},type="scan"}) by (instance) / sum(tikv_raftstore_region_count{{label_cluster_id},  type="region"}) by (instance)) *100,"name","tikv_memory_delta","","")'
  component: TiKV

- name: Seek_tombstone:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_coprocessor_scan_details{{label_cluster_id}, req="index",tag="seek_tombstone" }[2m])) ,"name","total","","")'
  component: TiDB
 
- name: CPS_stmt:by_type
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_server_query_total{{label_cluster_id}, type=~"StmtPrepare|StmtClose|StmtExecute|StmtReset|Query"}[2m]))by(type),"name","$1","type","(.*)")'
  component: TiDB

- name: grpc_request:by_type
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{{label_cluster_id}}[2m])) by (type),"name","$1","type","(.*)")'
  component: TiKV
  
- name: QPS_withError:total
  unit: "s"
  expr: 'label_replace(sum(sum(rate(tidb_tikvclient_region_err_total{{label_cluster_id}}[2m])) by (type)) + sum(sum(rate(tidb_tikvclient_backoff_seconds_count{{label_cluster_id}}[2m])) by (type)) + sum(sum(rate(tidb_tikvclient_lock_resolver_actions_total{{label_cluster_id}}[2m])) by (type)) + sum(sum(rate(tidb_tikvclient_lock_cleanup_task_total{{label_cluster_id}}[2m])) by (type)) ,"name","total","","")'
  component: TiDB
  
- name: TiKV_Error:total
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_tikvclient_region_err_total{{label_cluster_id}}[2m])) ,"name","total","","")'
  component: TiDB

#- name: Gc_lifetime:average
- name: Gc_lifetime_average:total
  unit: "1"
  expr: 'label_replace(max(tidb_tikvclient_gc_config{{label_cluster_id}, type="tikv_gc_life_time"}),"name","lifetime","","")'
  component: TiDB
  
#- name: Gc_run_interval:average
- name: Gc_run_interval_average:total
  unit: "1"
  expr: 'label_replace(max(tidb_tikvclient_gc_config{{label_cluster_id}, type="tikv_gc_run_interval"}),"name","lifetime","","")'
  component: TiDB

- name: Stats_autoanalyze_OPS:total
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_statistics_auto_analyze_total{{label_cluster_id}}[2m])) ,"name","count","","")'
  component: TiDB

- name: Copr_Scan_for_analyze_OPS:total
  unit: "1"
  expr: 'label_replace(sum(sum(rate(tikv_coprocessor_scan_keys_sum{{label_cluster_id}, req=~"analyze.*"}[2m])) by (req)) ,"name","count","","")'
  component: TiKV
  
- name: Copr_Scan_for_OPS:total
  unit: "1"
  expr: 'label_replace(sum(sum(rate(tikv_coprocessor_scan_keys_sum{{label_cluster_id}}[2m])) by (req)) ,"name","count","","")'
  component: TiKV

- name: Backup_CPU_Util:total
  unit: "%"
  expr: 'label_replace(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"b.*k.*w.*k.*"}[2m])) *100 ,"name","Util","","")'
  component: TiKV
  
- name: Backup_Thread_Count:total
  unit: "1"
  expr: 'label_replace(avg(sum(tikv_backup_thread_pool_size{{label_cluster_id}}) by(instance)) ,"name","count","","")'
  component: TiKV

- name: Copr_Scan_for_checksum_index_table_OPS:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_coprocessor_scan_keys_sum{{label_cluster_id}, req=~"checksum_(table|index)"}[2m])),"name","count","","")'
  component: TiKV

- name: PD_Scheduler_operator_create:total
  unit: "1"
  expr: 'label_replace(sum(delta(pd_schedule_operators_count{{label_cluster_id},  event="create"}[2m])) ,"name","total","","")'
  component: PD
  
- name: PD_Scheduler_operator_finish:total
  unit: "1"
  expr: 'label_replace(sum(delta(pd_schedule_operators_count{{label_cluster_id},  event="finish"}[2m])) ,"name","total","","")'
  component: PD
  
- name: PD_Scheduler_operator_timeout:total
  unit: "1"
  expr: 'label_replace(sum(delta(pd_schedule_operators_count{{label_cluster_id},  event="timeout"}[2m])) ,"name","total","","")'
  component: PD

- name: PD_Balance_leader_movement:total
  unit: "1"
  expr: 'label_replace(sum(sum(delta(pd_scheduler_balance_leader{{label_cluster_id},  type="move-leader"}[2m])) by (store) ) ,"name","total","","")'
  component: PD
  
- name: PD_Balance_region_movement:total
  unit: "1"
  expr: 'label_replace(sum(sum(delta(pd_scheduler_balance_region{{label_cluster_id},  type="move-peer"}[2m])) by (store)) ,"name","total","","")'
  component: PD
  
- name: PD_running_scheduler:total
  unit: "1"
  expr: 'label_replace(sum(pd_scheduler_status{{label_cluster_id}, type="allow"}) ,"name","total","","")'
  component: PD

- name: DDL_waiting:total
  unit: "1"
  expr: 'label_replace(sum(tidb_ddl_waiting_jobs{{label_cluster_id}}) ,"name","total","","")'
  component: TiDB

- name: DDL_add_idx_rate:total
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_ddl_add_index_total{{label_cluster_id},type="add_idx_rate"}[2m])) by (type) ,"name","total","","")'
  component: TiDB
  
- name: kv_Compaction_operations:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_engine_event_total{{label_cluster_id}, instance=~".*", db="kv"}[2m])) ,"name","total","","")'
  component: TiKV
  
- name: raft_Compaction_operations:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_engine_event_total{{label_cluster_id}, instance=~".*", db="raft"}[2m])) ,"name","total","","")'
  component: TiKV
  
- name: kv_Compaction_flows:total
  unit: "Bytes"
  expr: 'label_replace(sum(rate(tikv_engine_compaction_flow_bytes{{label_cluster_id}, instance=~".*", db="kv", type="bytes_read"}[2m])) + sum(rate(tikv_engine_compaction_flow_bytes{{label_cluster_id},  db="kv", type="bytes_written"}[2m])) + sum(rate(tikv_engine_flow_bytes{{label_cluster_id},  db="kv", type="flush_write_bytes"}[2m])) ,"name","total","","")'
  component: TiKV
  
- name: raft_Compaction_flows:total
  unit: "Bytes"
  expr: 'label_replace(sum(rate(tikv_engine_compaction_flow_bytes{{label_cluster_id}, instance=~".*", db="raft", type="bytes_read"}[2m])) + sum(rate(tikv_engine_compaction_flow_bytes{{label_cluster_id},  db="raft", type="bytes_written"}[2m])) + sum(rate(tikv_engine_flow_bytes{{label_cluster_id},  db="raft", type="flush_write_bytes"}[2m])) ,"name","total","","")'
  component: TiKV
  
- name: kv_Compaction_pending_bytes:total
  unit: "Bytes"
  expr: 'label_replace(sum(tikv_engine_pending_compaction_bytes{{label_cluster_id}, instance=~".*", db="kv"}),"name","total","","")'
  component: TiKV
  
- name: raft_Compaction_pending_bytes:total
  unit: "Bytes"
  expr: 'label_replace(sum(tikv_engine_pending_compaction_bytes{{label_cluster_id}, instance=~".*", db="raft"}),"name","total","","")'
  component: TiKV

- name: kv_write_stall_duration:total
  unit: "1"
  expr: 'label_replace(avg(tikv_engine_write_stall{{label_cluster_id}, instance=~".*", db="kv", type="write_stall_average"}) ,"name","total","","")'
  component: TiKV
  
- name: raft_write_stall_duration:total
  unit: "1"
  expr: 'label_replace(avg(tikv_engine_write_stall{{label_cluster_id}, instance=~".*", db="raft", type="write_stall_average"}) ,"name","total","","")'
  component: TiKV
  
- name: kv_write_stall_reason:total
  unit: "1"
  expr: 'label_replace(sum(increase(tikv_engine_write_stall_reason{{label_cluster_id}, instance=~".*", db="kv"}[2m]))  ,"name","total","","")'
  component: TiKV
  
- name: raft_write_stall_reason:total
  unit: "1"
  expr: 'label_replace(sum(increase(tikv_engine_write_stall_reason{{label_cluster_id}, instance=~".*", db="raft"}[2m]))  ,"name","total","","")'
  component: TiKV

- name: P99_coprocessor_handle_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{{label_cluster_id}}[2m])) by (le)),"name","total","","")'
  component: TiKV

- name: P99_coprocessor_wait_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{{label_cluster_id}}[2m])) by (le)),"name","total","","")'
  component: TiKV

- name: time_used_by_level_avg:total
  unit: "s"
  expr: 'label_replace(avg(sum(rate(tikv_multilevel_level_elapsed{{label_cluster_id},name="unified-read-pool"}[2m]))by (level)),"name","total","","")'
  component: TiKV

- name: unified_read_pool_running_tasks:total
  unit: "1"
  expr: 'label_replace(avg(sum(avg_over_time(tikv_unified_read_pool_running_tasks{{label_cluster_id}}[2m])) by (instance)) ,"name","total","","")'
  component: TiKV

- name: Latch_wait_duration_prewrite:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{{label_cluster_id}, instance=~".*", type="prewrite"}[2m])) by (le)) ,"name","total","","")'
  component: TiKV

- name: Latch_wait_duration_commit:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{{label_cluster_id}, instance=~".*", type="commit"}[2m])) by (le)) ,"name","total","","")'
  component: TiKV
  
- name: Latch_command_duration_prewrite:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{{label_cluster_id}, instance=~".*", type="prewrite"}[2m])) by (le)) ,"name","total","","")'
  component: TiKV

- name: Latch_command_duration_commit:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{{label_cluster_id}, instance=~".*", type="commit"}[2m])) by (le)) ,"name","total","","")'
  component: TiKV

#- name: scheduler_pending_command:max
- name: scheduler_pending_command_max:total
  unit: "1"
  expr: 'label_replace(max(sum(tikv_scheduler_contex_total{{label_cluster_id}}) by (instance)) ,"name","total","","")'
  component: TiKV

- name: scheduler_writing_bytes_max:total
  unit: "Bytes"
  expr: 'label_replace(max(sum(tikv_scheduler_writing_bytes{{label_cluster_id}}) by (instance)) ,"name","total","","")'
  component: TiKV

- name: Avg_async_snapshot_duration:total
  unit: "s"
  expr: 'label_replace(sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{{label_cluster_id},type="snapshot"}[2m]))/sum(rate(tikv_storage_engine_async_request_duration_seconds_count{{label_cluster_id},type="snapshot"}[2m])),"name","total","","")'
  component: TiKV

- name: Local_reader_reject_pct:total
  unit: "%"
  expr: 'label_replace((avg(topk(1,sum(rate(tikv_raftstore_local_read_reject_total{{label_cluster_id}}[2m])) by  (reason))) /sum(rate(tikv_raftstore_local_read_executed_requests{{label_cluster_id}}[2m]))) *100, "name","Pct","","")'
  component: TiKV
 
#- name: number_of_snapshot:avg
- name: number_of_snapshot_avg:total
  unit: "1"
  expr: 'label_replace(avg(tikv_engine_num_snapshots{{label_cluster_id},  db="kv"}) ,"name","avg","","")'
  component: TiKV

- name: snapshot_actions:total
  unit: "1"
  expr: 'label_replace(sum(delta(tikv_raftstore_snapshot_total{{label_cluster_id}}[2m]))  ,"name","avg","","")'
  component: TiKV
  
- name: P99_propose_wait_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99,sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{{label_cluster_id}}[2m]))by(le)),"name","total","","")'
  component: TiKV 
 
- name: P99_apply_wait_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99,sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{{label_cluster_id}}[2m]))by(le)),"name","total","","")'
  component: TiKV 

- name: region_count_per_tikv:total
  unit: "1"
  expr: 'label_replace(avg(sum(tikv_raftstore_region_count{{label_cluster_id},  type="region"}) by (instance)),"name","total","","")'
  component: TiKV

- name: leader_count_per_tikv:total
  unit: "1"
  expr: 'label_replace(avg(pd_scheduler_store_status{{label_cluster_id}, type="leader_count",address=~".*tikv.*"}) by (store),"name","total","","")'
  component: TiKV

- name: region_heartbeat_Report:total
  unit: "1"
  expr: 'label_replace(avg(sum(delta(pd_scheduler_region_heartbeat{{label_cluster_id}, type="report", status="ok"}[2m])) by (address, store)) ,"name","total","","")'
  component: TiKV

- name: update_cache_count:total
  unit: "1"
  expr: 'label_replace(sum(rate(pd_cluster_region_event{{label_cluster_id},event="update_cache"}[2m])) by (address, store, event),"name","total","","")'
  component: TiKV
  
- name: active_writting_leaders:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(tikv_region_written_keys_count{{label_cluster_id}}[2m])) by (instance)),"name","total","","")'
  component: TiKV 

- name: awaken:total
  unit: "1"
  expr: 'label_replace(avg(sum(tikv_raftstore_hibernated_peer_state{{label_cluster_id},  state="awaken"}) by (instance, state) ) by (state),"name","total","","")'
  component: TiKV
  
- name: hibernated:total
  unit: "1"
  expr: 'label_replace(avg(sum(tikv_raftstore_hibernated_peer_state{{label_cluster_id},  state="hibernated"}) by (instance, state) ) by (state),"name","total","","")'
  component: TiKV

- name: heartbeat_message:total
  unit: "1"
  expr: 'label_replace(sum(sum(rate(tikv_raftstore_raft_sent_message_total{{label_cluster_id}, type=~"heartbeat|heartbeat_resp"}[2m])) by (type)),"name","total","","")'
  component: TiKV
  
- name: all_message:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_raftstore_raft_sent_message_total{{label_cluster_id}}[2m])),"name","total","","")'
  component: TiKV
  
- name: pseudo_estimation_total:total
  unit: "1"
  expr: 'label_replace(sum(sum(rate(tidb_statistics_pseudo_estimation_total{{label_cluster_id}}[2m])) by (type)),"name","total","","")'
  component: TiDB

- name: stats_unhealthy_pct:total
  unit: "%"
  expr: 'label_replace((sum(avg(tidb_statistics_stats_healthy{{label_cluster_id},type!="[100,100]"}) by (type))/ sum(avg(tidb_statistics_stats_healthy{{label_cluster_id}}) by (type)))*100,"name","pct","","")'
  component: TiDB
  
- name: internal_QPS:total
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_session_restricted_sql_total{{label_cluster_id}}[2m])),"name","total","","")'
  component: TiDB

#- name: TiFlash_memory_usage:avg
- name: TiFlash_memory_usage_avg:total
  unit: "Bytes"
  expr: 'label_replace(avg(tiflash_proxy_process_resident_memory_bytes{{label_cluster_id}, {label_component}="tiflash"}),"name","total","","")'
  component: TiFlash
  
#- name: TiFlash_memory_usage:max
- name: TiFlash_memory_usage_max:total
  unit: "Bytes"
  expr: 'label_replace(max(tiflash_proxy_process_resident_memory_bytes{{label_cluster_id}, {label_component}="tiflash"}),"name","total","","")'
  component: TiFlash
  
#- name: TiFlash_memory_usage:delta
- name: TiFlash_memory_usage_delta:total
  unit: "Bytes"
  expr: 'label_replace(max(tiflash_proxy_process_resident_memory_bytes{{label_cluster_id}, {label_component}="tiflash"}) - min(tiflash_proxy_process_resident_memory_bytes{{label_cluster_id}, {label_component}="tiflash"}),"name","total","","")'
  component: TiFlash

#- name: TiFlash_CPU_usage:avg
- name: TiFlash_CPU_usage_avg:total
  unit: "%"
  expr: 'label_replace(avg(rate(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"}[2m]))*100,"name","total","","")'
  component: TiFlash

#- name: TiFlash_CPU_usage:max
- name: TiFlash_CPU_usage_max:total
  unit: "%"
  expr: 'label_replace(max(rate(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"}[2m]))*100,"name","total","","")'
  component: TiFlash
  
#- name: TiFlash_CPU_usage:delta
- name: TiFlash_CPU_usage_delta:total
  unit: "%"
  expr: 'label_replace((max(rate(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"}[2m])) - min(rate(tiflash_proxy_process_cpu_seconds_total{{label_cluster_id}, {label_component}="tiflash"}[2m])))*100 ,"name","total","","")'
  component: TiFlash

- name: TiFlash_disk_usage_avg:total
  unit: "%"
  expr: 'label_replace(avg(rate(tiflash_system_current_metric_StoreSizeUsed{{label_cluster_id}[2m]))*100,"name","total","","")'
  component: TiFlash

- name: TiFlash_disk_usage_max:total
  unit: "%"
  expr: 'label_replace(max(rate(tiflash_system_current_metric_StoreSizeUsed{{label_cluster_id}[2m]))*100,"name","total","","")'
  component: TiFlash

- name: TiFlash_disk_usage_delta:total
  unit: "%"
  expr: 'label_replace((max(rate(tiflash_system_current_metric_StoreSizeUsed{{label_cluster_id}[2m])) - min(rate(tiflash_system_current_metric_StoreSizeUsed{{label_cluster_id}}[2m])))*100 ,"name","total","","")'
  component: TiFlash

- name: kv_write_operation_timeout:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_engine_write_served{{label_cluster_id},  db="kv", type="write_timeout"}[2m])),"name","total","","")'
  component: TiKV
  
- name: P99_kv_get_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_get_micro_seconds{{label_cluster_id},  db="kv",type="get_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_kv_seek_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_seek_micro_seconds{{label_cluster_id},  db="kv",type="seek_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_kv_write_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_write_micro_seconds{{label_cluster_id},  db="kv",type="write_percentile99"})/1000000,"name","total","","")'
  component: TiKV

- name: P99_kv_write_wal_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_write_wal_time_micro_seconds{{label_cluster_id},  db="kv",type="write_wal_micros_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_kv_wal_sync_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_wal_file_sync_micro_seconds{{label_cluster_id},  db="kv",type="wal_file_sync_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: raft_write_operation_timeout:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_engine_write_served{{label_cluster_id},  db="raft", type="write_timeout"}[2m])),"name","total","","")'
  component: TiKV
  
- name: P99_raft_get_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_get_micro_seconds{{label_cluster_id},  db="raft",type="get_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_raft_seek_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_seek_micro_seconds{{label_cluster_id},  db="raft",type="seek_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_raft_write_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_write_micro_seconds{{label_cluster_id},  db="raft",type="write_percentile99"})/1000000,"name","total","","")'
  component: TiKV

- name: P99_raft_write_wal_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_write_wal_time_micro_seconds{{label_cluster_id},  db="raft",type="write_wal_micros_percentile99"})/1000000,"name","total","","")'
  component: TiKV
  
- name: P99_raft_wal_sync_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_wal_file_sync_micro_seconds{{label_cluster_id},  db="raft",type="wal_file_sync_percentile99"})/1000000,"name","total","","")'
  component: TiKV

- name: P99_raft_engine_write_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99,sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{{label_cluster_id}}[2m]))by(le)),"name","total","","")'
  component: TiKV

- name: P999_raft_engine_sync_log_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.999,sum(rate(raft_engine_write_leader_duration_seconds_bucket{{label_cluster_id}}[2m]))by(le)),"name","total","","")'
  component: TiKV

- name: P99_kv_bytes_write:total
  unit: "Bytes"
  expr: 'label_replace(avg(tikv_engine_bytes_per_write{{label_cluster_id},  db="kv",type="bytes_per_write_percentile99"}),"name","total","","")'
  component: TiKV
  
- name: P99_raft_bytes_write:total
  unit: "Bytes"
  expr: 'label_replace(avg(tikv_engine_bytes_per_write{{label_cluster_id},  db="raft",type="bytes_per_write_percentile99"}),"name","total","","")'
  component: TiKV
  
- name: raftstore_proposal_total:total
  unit: "Bytes"
  expr: 'label_replace(avg(sum(rate(tikv_raftstore_proposal_total{{label_cluster_id}, type="normal"}[2m])) by (instance)),"name","total","","")'
  component: TiKV

- name: sst_read_duration:total
  unit: "s"
  expr: 'label_replace(avg(tikv_engine_sst_read_micros{{label_cluster_id}, db="kv", type="sst_read_micros_percentile99"})/1000000,"name","total","","")'
  component: TiKV

- name: admin_apply_remove_peer:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_raftstore_admin_cmd_total{{label_cluster_id},  status="success", type!="compact", type="remove_peer"}[2m]))  by (type) ,"name","total","","")'
  component: TiKV

- name: kv_memtable_hit:total
  unit: "%"
  expr: 'label_replace((sum(rate(tikv_engine_memtable_efficiency{{label_cluster_id},  db="kv", type="memtable_hit"}[2m])) / (sum(rate(tikv_engine_memtable_efficiency{{label_cluster_id}, db="kv", type="memtable_hit"}[2m])) + sum(rate(tikv_engine_memtable_efficiency{{label_cluster_id}, db="kv", type="memtable_miss"}[2m]))))*100 ,"name","pct","","")'
  component: TiKV
  
- name: kv_blockcache_hit:total
  unit: "%"
  expr: 'label_replace((sum(rate(tikv_engine_cache_efficiency{{label_cluster_id},  db="kv", type="block_cache_hit"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{{label_cluster_id},  db="kv", type="block_cache_hit"}[2m])) + sum(rate(tikv_engine_cache_efficiency{{label_cluster_id},  db="kv", type="block_cache_miss"}[2m])))) *100 ,"name","Util","","")'
  component: TiKV
  
- name: abnormal_store:total
  unit: "1"
  expr: 'label_replace(sum(pd_cluster_status{{label_cluster_id},type="store_disconnected_count"}) + sum(pd_cluster_status{{label_cluster_id},  type="store_unhealth_count"}) + sum(pd_cluster_status{{label_cluster_id},  type="store_low_space_count"}) + sum(pd_cluster_status{{label_cluster_id},  type="store_down_count"}) + sum(pd_cluster_status{{label_cluster_id},  type="store_offline_count"}) + sum(pd_cluster_status{{label_cluster_id},  type="store_tombstone_count"}),"name","total","","")'
  component: TiKV

- name: raft_message_for_vote:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(tikv_raftstore_raft_sent_message_total{{label_cluster_id},  type="vote"}[2m])) by (instance)),"name","total","","")'
  component: TiKV
  
- name: working_pending_tasks:total
  unit: "1"
  expr: 'label_replace(sum(rate(tikv_worker_pending_task_total{{label_cluster_id}}[2m])) ,"name","total","","")'
  component: TiKV

- name: unhealthy_regions:total
  unit: "1"
  expr: 'label_replace(sum(pd_regions_status{{label_cluster_id}, type!="learner-peer-region-count"}) ,"name","total","","")'
  component: TiKV

- name: P99_pd_server_handle_tso_duration:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{{label_cluster_id}}[2m])) by (type, le)),"name","total","","")'
  component: TiKV

- name: P99_Region_heartbeat_handle_latency:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99, sum(rate(pd_scheduler_handle_region_heartbeat_duration_seconds_bucket{{label_cluster_id}}[2m])) by (le)),"name","total","","")'
  component: TiKV
  
- name: TiDB_CPU_avg:total
  unit: "%"
  expr: 'label_replace(avg(rate(process_cpu_seconds_total{{label_cluster_id},{label_component}="tidb"}[2m]))*100,"name","total","","")'
  component: TiDB
  
- name: TiDB_memory_avg:total
  unit: "Bytes"
  expr: 'label_replace(avg(process_resident_memory_bytes{{label_cluster_id},{label_component}="tidb"}),"name","total","","")'
  component: TiDB
  
- name: TiKV_memory_avg:total
  unit: "Bytes"
  expr: 'label_replace(avg(process_resident_memory_bytes{{label_cluster_id},{label_component}="tikv"}),"name","tikv_memory_avg","","")'
  component: TiKV

- name: TiKV_CPU_avg:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id}}[2m]))by(instance))*100,"name","tikv_cpu_max","","")'
  component: TiKV

- name: unified_read_pool_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"unified_read_po*"}[2m])) by (instance))*100,"name","unified_read_pool_cpu","","")'
  component: TiKV
  
- name: gc_worker_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"gc_worker.*"}[2m])) by (instance))*100,"name","gc_worker_cpu","","")'
  component: TiKV

- name: raftstore_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"raftstore_.*"}[2m])) by (instance))*100,"name","raftstore_cpu","","")'
  component: TiKV
  
- name: async_apply_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"apply_[0-9]+"}[2m])) by (instance))*100,"name","async_apply_cpu","","")'
  component: TiKV

- name: scheduler_worker_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"sched_.*"}[2m])) by (instance))*100,"name","scheduler_worker_cpu","","")'
  component: TiKV
  
- name: grpc_pool_cpu:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"grpc.*"}[2m])) by (instance))*100,"name","grpc_pool_cpu","","")'
  component: TiKV

- name: rocksdb_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"rocksdb.*"}[2m])) by (instance))*100,"name","rocksdb_cpu","","")'
  component: TiKV
  
- name: background_worker_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"background.*"}[2m])) by (instance))*100,"name","background_worker_cpu","","")'
  component: TiKV

- name: backup_worker_cpu:total
  unit: "%"
  expr: 'label_replace(avg(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id},  name=~"(backup-worker|bkwkr).*"}[2m])) by (instance))*100,"name","backup_worker_cpu","","")'
  component: TiKV
  
- name: TiKV_disk_usage_avg:total
  unit: "%"
  expr: 'label_replace(avg(sum(tikv_engine_size_bytes{{label_cluster_id}})by(instance)/sum(tikv_store_size_bytes{{label_cluster_id},type="capacity"})by(instance))*100,"name","tikv_disk_usage_avg","","")'
  component: TiKV
  
#- name: TiKV_store_available_ratio_avg
- name: TiKV_store_available_ratio_avg:total
  unit: "%"
  expr: 'label_replace( avg(sum(pd_scheduler_store_status{{label_cluster_id}, address=~"db-tikv.*", type="store_available"}) by (address, store) / sum(pd_scheduler_store_status{{label_cluster_id}, address=~"db-tikv.*",  type="store_capacity"}) by (address, store))*100,"name","store_available_ratio_avg","","")'
  component: TiKV
  
#- name: TiKV_store_available_ratio_max
- name: TiKV_store_available_ratio_max:total
  unit: "%"
  expr: 'label_replace( max(sum(pd_scheduler_store_status{{label_cluster_id}, address=~"db-tikv.*", type="store_available"}) by (address, store) / sum(pd_scheduler_store_status{{label_cluster_id}, address=~"db-tikv.*",  type="store_capacity"}) by (address, store))*100,"name","store_available_ratio_max","","")'
  component: TiKV
  
- name: TiKV_MBps_avg:total
  unit: "Bps"
  expr: 'label_replace(avg(sum(rate(node_disk_read_bytes_total{{label_cluster_id}}[2m]) + rate(node_disk_written_bytes_total{{label_cluster_id}}[2m])) by(instance)),"name","TiKV_MBps_avg","","")'
  component: TiKV

- name: TiKV_MBps_max:total
  unit: "Bps"
  expr: 'label_replace(max(sum(rate(node_disk_read_bytes_total{{label_cluster_id}}[2m]) + rate(node_disk_written_bytes_total{{label_cluster_id}}[2m])) by(instance)),"name","TiKV_MBps_max","","")'
  component: TiKV
  
- name: TiKV_MBps_delta:total
  unit: "Bps"
  expr: 'label_replace((max(sum(rate(node_disk_read_bytes_total{{label_cluster_id}}[2m]) + rate(node_disk_written_bytes_total{{label_cluster_id}}[2m])) by(instance)) - min(sum(rate(node_disk_read_bytes_total{{label_cluster_id}}[2m]) + rate(node_disk_written_bytes_total{{label_cluster_id}}[2m])) by(instance))),"name","TiKV_MBps_delta","","")'
  component: TiKV
  
- name: TiKV_IOPS_avg:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(node_disk_reads_completed_total{{label_cluster_id}}[2m]) + rate(node_disk_writes_completed_total{{label_cluster_id}}[2m])) by(instance)),"name","TiKV_IOPS_avg","","")'
  component: TiKV

- name: TiKV_IOPS_max:total
  unit: "1"
  expr: 'label_replace(max(sum(rate(node_disk_reads_completed_total{{label_cluster_id}}[2m]) + rate(node_disk_writes_completed_total{{label_cluster_id}}[2m])) by(instance)),"name","TiKV_IOPS_max","","")'
  component: TiKV
  
- name: TiKV_MBps_delta:total
  unit: "1"
  expr: 'label_replace((max(sum(rate(node_disk_reads_completed_total{{label_cluster_id}}[2m]) + rate(node_disk_writes_completed_total{{label_cluster_id}}[2m])) by(instance)) - min(sum(rate(node_disk_reads_completed_total{{label_cluster_id}}[2m]) + rate(node_disk_writes_completed_total{{label_cluster_id}}[2m])) by(instance))),"name","TiKV_IOPS_delta","","")'
  component: TiKV
 
- name: TiDB_client_network_avg:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(tidb_server_packet_io_bytes_sum{{label_cluster_id}}[2m])) by (instance)),"name","TiDB_client_network_avg","","")'
  component: TiKV

- name: TiDB_client_network_max:total
  unit: "1"
  expr: 'label_replace(max(sum(rate(tidb_server_packet_io_bytes_sum{{label_cluster_id}}[2m])) by (instance)),"name","TiDB_client_network_max","","")'
  component: TiKV
  
- name: TiDB_client_network_delta:total
  unit: "1"
  expr: 'label_replace((max(sum(rate(tidb_server_packet_io_bytes_sum{{label_cluster_id}}[2m])) by (instance)) - min(sum(rate(tidb_server_packet_io_bytes_sum{{label_cluster_id}}[2m])) by (instance))),"name","TiDB_client_network_delta","","")'
  component: TiKV

- name: TiKV_network_avg:total
  unit: "1"
  expr: 'label_replace(avg(sum(rate(node_network_receive_bytes_total{{label_cluster_id},job="node-exporter" }[2m]) + rate(node_network_transmit_bytes_total{{label_cluster_id},job="node-exporter"}[2m])) by (instance)),"name","TiKV_network_avg","","")'
  component: TiKV

- name: TiKV_network_max:total
  unit: "1"
  expr: 'label_replace(max(sum(rate(node_network_receive_bytes_total{{label_cluster_id},job="node-exporter" }[2m]) + rate(node_network_transmit_bytes_total{{label_cluster_id},job="node-exporter"}[2m])) by (instance)),"name","TiKV_network_max","","")'
  component: TiKV
  
- name: TiKV_network_delta:total
  unit: "1"
  expr: 'label_replace((max(sum(rate(node_network_receive_bytes_total{{label_cluster_id},job="node-exporter" }[2m]) + rate(node_network_transmit_bytes_total{{label_cluster_id},job="node-exporter"}[2m])) by (instance)) - min(sum(rate(node_network_receive_bytes_total{{label_cluster_id},job="node-exporter" }[2m]) + rate(node_network_transmit_bytes_total{{label_cluster_id},job="node-exporter"}[2m])) by (instance))),"name","TiKV_network_delta","","")'
  component: TiKV
  
- name: P99_query_duration_all:total
  unit: "s"
  expr: 'label_replace(histogram_quantile(0.99,sum(rate(tidb_server_handle_query_duration_seconds_bucket{{label_cluster_id}}[2m]))by(le)),"name","total","","")'
  component: TiDB
  
- name: table_count_by_sum_stats_healthy:total
  unit: "1"
  expr: 'label_replace(sum(avg(tidb_statistics_stats_healthy{{label_cluster_id}}) by (type)),"name","total","","")'
  component: TiDB
  
- name: KV_request_OPS:total
  unit: "1"
  expr: 'label_replace(sum(rate(tidb_tikvclient_request_seconds_count{{label_cluster_id}}[2m])),"name","total","","")'
  component: TiDB