---
raw:
cooked:
  - id: 98
    name: Conn idle duration
    method: anomaly
    object:
      record: conn_idle_duration:in_txn
      expr: 'label_replace((sum(rate(tidb_server_conn_idle_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", in_txn="1"}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", in_txn="1"}[1m]))),"name","conn_idle_in_txn","","")'
    targets:
    - record: conn_idle_duration:not_in_txn
      expr: 'label_replace((sum(rate(tidb_server_conn_idle_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", in_txn="0"}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", in_txn="0"}[1m]))),"name","conn_idle_not_in_txn","","")'

  - id: 100
    name: connection count
    method: anomaly
    object:
      record: conn_total
      expr: 'label_replace(sum(tidb_server_connections{cluster_id=~".*{tidb_cluster_id}"}),"name","conn_total","","")'
    targets:
    - record: conn_active
      expr: 'label_replace(sum(rate(tidb_server_handle_query_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", sql_type!="internal"}[1m])),"name","conn_act","","")'

  - id: 102
    name: QPS
    method: anomaly
    object:
      record: qps_total
      expr: 'label_replace(sum(rate(tidb_executor_statement_total{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","qps_total","","")'
    targets:
    - record: qps_failed
      expr: 'label_replace(sum(rate(tidb_server_execute_error_total{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","qps_failed","","")'
      
  - id: 2
    name: Total Latency
    method: anomaly
    object:
      # total latency from query to response p99
      # histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{cluster_id=~".*{tidb_cluster_id}", sql_type!="internal"}[1m])) by (le))
      # internal is DDL?
      record: tidb_p99_rt:total
      expr: 'label_replace(sum(rate(tidb_server_handle_query_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", sql_type!="internal"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", sql_type!="internal"}[1m])),"name","total_latency","","")'
    targets:

  - id: 30
    name: Get token duration
    method: anomaly
    object:
      record: get_token_duration
      expr: 'label_replace(sum(rate(tidb_server_get_token_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tidb_server_get_token_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","get_token_duration","","")'
    targets:

  - id: 32
    name: Parse Duration
    method: anomaly
    object: 
      record: parse_duration:general
      expr: 'label_replace((sum(rate(tidb_session_parse_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m])) / sum(rate(tidb_session_parse_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m]))),"name","parse_duration","","")'
    targets:

  - id: 34
    name: Compile duration
    method: anomaly
    object: 
      record: compile_duration:general
      expr: 'label_replace((sum(rate(tidb_session_compile_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m])) / sum(rate(tidb_session_compile_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m]))),"name","compile_duration","","")'
    targets:

  - id: 111
    name: Plan cache and stmtexecute
    method: compare
    object: 
      record: plan_cache
      expr: 'label_replace(sum(rate(tidb_server_plan_cache_total{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","plan_cache","","")'
    targets:
    - record: stmt_execute
      expr: 'label_replace(sum(rate(tidb_server_query_total{cluster_id=~".*{tidb_cluster_id}", type="StmtExecute"}[1m])),"name","stmt_execute","","")'

  - id: 108
    name: cps
    method: compare
    object: 
      record: cps_total
      expr: 'label_replace(sum(rate(tidb_server_query_total{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","cps_total","","")'
    targets:
    - record: stmt_execute
      expr: 'label_replace(sum(rate(tidb_server_query_total{cluster_id=~".*{tidb_cluster_id}", type="StmtExecute"}[1m])),"name","stmt_execute","","")'

  - id: 36
    name: Execute duration
    method: anomaly
    object: 
      record: execute_duration:general
      expr: 'label_replace((sum(rate(tidb_session_execute_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m])) / sum(rate(tidb_session_execute_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", sql_type="general"}[1m]))),"name","execute_duration","","")'
    targets:

  - id: 40
    name: Wait start tso duration
    method: anomaly
    object: 
      record: wait_start_tso_duration
      expr: 'label_replace((sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type="wait"}[1m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type="wait"}[1m]))),"name","wait_start_tso_duration","","")'
    targets:

  - id: 56
    name: KV backoff duration
    method: anomaly
    object: 
      record: kv_backoff_duration
      expr: 'label_replace(sum(rate(tidb_tikvclient_backoff_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tidb_tikvclient_backoff_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","kv_backoff_duration","","")'
    targets:

  - id: 38
    name: KV request duration
    method: anomaly
    object: 
      record: kv_request_duration
      expr: 'label_replace(sum(rate(tidb_tikvclient_request_seconds_sum{cluster_id=~".*{tidb_cluster_id}", store!="0"}[1m])) / sum(rate(tidb_tikvclient_request_seconds_count{cluster_id=~".*{tidb_cluster_id}", store!="0"}[1m])),"name","kv_request_duration","","")'
    targets:

  - id: 4
    name: TiKV Latency
    method: anomaly
    object:
    # total latency of TiKV (all type of gRPC message duration)
    # histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{cluster_id=~".*{tidb_cluster_id}"}[1m])) by (le))
    # how about ti-flash?
      record: tikv_p99_rt:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","tikv_latency","","")'
    targets:

  - id: 42
    name: Coprocessor gRPC duration
    method: anomaly
    object: 
      record: coprocessor_duration:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])),"name","coprocessor_duration","","")'
    targets:
    - record: coprocessor_duration:by_instance
      expr: (sum(rate(tikv_grpc_msg_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])) by (instance)) / (sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])) by (instance))

  - id: 44
    name: Get&Batch-Get duration
    method: anomaly
    object: 
      record: small_read_duration:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type=~"coprocessor"}[1m])),"name","small_read_duration","","")'
    targets:
    - record: small_read_duration:by_instance
      expr: (sum(rate(tikv_grpc_msg_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type=~"kv_batch_get|kv_batch_get_command|kv_get|kv_scan"}[1m])) by (instance)) / (sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type=~"kv_batch_get|kv_batch_get_command|kv_get|kv_scan"}[1m])) by (instance))
  
  - id: 24
    name: TiKV Write Duration
    method: anomaly
    object:
      record: write_rpc_latency
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_sum{type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])),"name","tikv_write_latency","","")'
    targets:
    - record: write_rpc_duration:by_instance
      expr: (sum(rate(tikv_grpc_msg_duration_seconds_sum{type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])) by (instance)) / (sum(rate(tikv_grpc_msg_duration_seconds_count{type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])) by (instance))

  - id: 54
    name: Coprocessor wait duration
    method: anomaly
    object: 
      record: copr_wait_duration
      expr: 'label_replace(sum(rate(tikv_coprocessor_request_wait_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","copr_wait_duration","","")'
    targets:

  - id: 58
    name: Coprocessor handle duration
    method: anomaly
    object: 
      record: copr_handle_duration
      expr: 'label_replace(sum(rate(tikv_coprocessor_request_handle_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","copr_handle_duration","","")'
    targets:

  - id: 48
    name: Latch wait duration
    method: anomaly
    object: 
      record: latch_wait_duration
      expr: 'label_replace(sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","latch_wait_duration","","")'
    targets:

  - id: 52
    name: Async snapshot duration
    method: anomaly
    object: 
      record: async_snapshot_duration
      expr: 'label_replace(sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type="snapshot"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type="snapshot"}[1m])),"name","async_snapshot_duration","","")'
    targets:

  - id: 112
    name: Async write duration
    method: anomaly
    object: 
      record: async_write_duration
      expr: 'label_replace(sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{cluster_id=~".*{tidb_cluster_id}", type="write"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}", type="write"}[1m])),"name","async_write_duration","","")'
    targets:

  - id: 35
    name: Store duration
    method: anomaly
    object: 
      record: store
      expr: 'label_replace(sum(rate(tikv_raftstore_store_duration_secs_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_raftstore_store_duration_secs_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","store_duration","","")'
    targets:

  - id: 110
    name: Apply duration
    method: anomaly
    object: 
      record: apply
      expr: 'label_replace((sum(rate(tikv_raftstore_apply_duration_secs_sum{cluster_id=~".*{tidb_cluster_id}"}[1m])) / sum(rate(tikv_raftstore_apply_duration_secs_count{cluster_id=~".*{tidb_cluster_id}"}[1m]))),"name","apply_duration","","")'
    targets:

  # gRPC
  - id: 12
    name: gRPC count
    method: compare
    object:
      record: grpc:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}"}[1m])),"name","total_rpc","","")'
    targets:
    - record: write-rpc:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])),"name","write_rpc","","")'
    - record: copr-rpc:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type=~"kv_get|kv_scan|kv_batch_get|kv_batch_get_command"}[1m])),"name","copr_rpc","","")'
    - record: smallread-rpc:total
      expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type="coprocessor"}[1m])),"name","smallread_rpc","","")'

  - id: 60
    name: Write imbalance
    method: imbalance
    object:
      record: write-rpc:by_instance
      expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m]))by(instance)'
    targets:

  - id: 64
    name: Coprocessor imbalance
    method: imbalance
    object:
      record: copr-rpc:by_instance
      expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type="coprocessor"}[1m]))by(instance)'
    targets:

  - id: 62
    name: Small read imbalance
    method: imbalance
    object:
      record: small-read-rpc:by_instance
      expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{cluster_id=~".*{tidb_cluster_id}",type=~"kv_get|kv_scan|kv_batch_get|kv_batch_get_command"}[1m]))by(instance)'
    targets: