--- raw: cooked: # Imbalance write? - id: 2 name: Total Latency method: anomaly object: # total latency from query to response p99 # histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster="$tidb_cluster", sql_type!="internal"}[1m])) by (le)) # internal is DDL? record: tidb_p99_rt:total expr: 'label_replace(histogram_quantile(0.99,sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m]))by(le)),"name","total_latency","","")' targets: - id: 30 name: Get token duration method: anomaly object: record: get_token_duration expr: 'label_replace(histogram_quantile(0.99,sum(rate(tidb_server_get_token_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","get_token_duration","","")' targets: - id: 32 name: Parse Duration method: anomaly object: record: parse_duration:general expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", sql_type="general"}[1m])) by (le)),"name","parse_duration","","")' targets: - id: 34 name: Compile duration method: anomaly object: record: compile_duration:general expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", sql_type="general"}[1m])) by (le)),"name","compile_duration","","")' targets: - id: 36 name: Execute duration method: anomaly object: record: execute_duration:general expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", sql_type="general"}[1m])) by (le)),"name","execute_duration","","")' targets: - id: 40 name: Wait start tso duration method: anomaly object: record: wait_start_tso_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","wait_start_tso_duration","","")' targets: - id: 56 name: KV backoff duration method: anomaly object: record: kv_backoff_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[5m])) by (le)),"name","kv_backoff_duration","","")' targets: - id: 38 name: KV request duration method: anomaly object: record: kv_request_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{tidb_cluster="{tidb_cluster_id}", store!="0"}[1m])) by (le)),"name","kv_request_duration","","")' targets: - id: 4 name: TiKV Latency method: anomaly object: # Latency chain # total latency of TiKV (all type of gRPC message duration) # histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="$tidb_cluster"}[1m])) by (le)) # how about ti-flash? record: tikv_p99_rt:total expr: 'label_replace(histogram_quantile(0.99,sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m]))by(le))>0,"name","tikv_latency","","")' targets: - id: 42 name: Coprocessor gRPC duration method: anomaly object: record: coprocessor_duration:total expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type="coprocessor"}[1m]))by(le)),"name","coprocessor_duration","","")' targets: - record: coprocessor_duration:by_instance expr: histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type="coprocessor"}[1m]))by(le,instance)) - id: 44 name: Get&Batch-Get duration method: anomaly object: record: small_read_duration:total expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type=~"kv_batch_get|kv_batch_get_command|kv_get|kv_scan"}[1m]))by(le)),"name","small_read_duration","","")' targets: - record: small_read_duration:by_instance expr: histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type=~"kv_batch_get|kv_batch_get_command|kv_get|kv_scan"}[1m]))by(le,instance)) - id: 24 name: TiKV Write Duration method: anomaly object: record: write_rpc_latency expr: 'label_replace(histogram_quantile(0.99,sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m]))by(le)),"name","tikv_write_latency","","")' targets: - record: write_rpc_duration:by_instance expr: histogram_quantile(0.99,sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m]))by(le,instance)) - id: 54 name: Coprocessor wait duration method: anomaly object: record: copr_wait_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","copr_wait_duration","","")' targets: - id: 58 name: Coprocessor handle duration method: anomaly object: record: copr_handle_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","copr_handle_duration","","")' targets: - id: 46 name: Scheduler command duration method: anomaly object: record: scheduler_command_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","scheduler_command_duration","","")' targets: - id: 48 name: Latch wait duration method: anomaly object: record: latch_wait_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}"}[1m])) by (le)),"name","latch_wait_duration","","")' targets: - id: 52 name: Async snapshot duration method: anomaly object: record: async_snapshot_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type="snapshot"}[1m])) by (le)),"name","async_snapshot_duration","","")' targets: - id: 50 name: Async write duration method: anomaly object: record: async_write_duration expr: 'label_replace(histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{tidb_cluster="{tidb_cluster_id}", type="write"}[1m])) by (le)),"name","async_write_duration","","")' targets: # gRPC - id: 12 name: gRPC count method: compare object: record: grpc:total expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}"}[1m])),"name","total_rpc","","")' targets: - record: write-rpc:total expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m])),"name","write_rpc","","")' - record: copr-rpc:total expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type=~"kv_get|kv_scan|kv_batch_get|kv_batch_get_command"}[1m])),"name","copr_rpc","","")' - record: smallread-rpc:total expr: 'label_replace(sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type="coprocessor"}[1m])),"name","smallread_rpc","","")' - id: 60 name: Write imbalance method: imbalance object: record: write-rpc:by_instance expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type=~"kv_prewrite|kv_commit|kv_pessimistic_lock"}[1m]))by(instance)' targets: - id: 64 name: Coprocessor imbalance method: imbalance object: record: copr-rpc:by_instance expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type="coprocessor"}[1m]))by(instance)' targets: - id: 62 name: Small read imbalance method: imbalance object: record: small-read-rpc:by_instance expr: 'sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster="{tidb_cluster_id}",type=~"kv_get|kv_scan|kv_batch_get|kv_batch_get_command"}[1m]))by(instance)' targets: