{ "dashboard_title": "Test-Cluster-TiKV-Details", "sections": [ { "title": "Duration", "panels": [ { "title": "Write Pipeline Duration", "subtitle": "Write Pipeline Composition", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_append_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Cop Read Duration", "subtitle": "Read Duration Composition", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.95,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] } ] }, { "title": "Cluster", "panels": [ { "title": "Store size", "subtitle": "The storage size per TiKV instance", "expressions": [ "sum(( tikv_store_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"} )) by (instance)" ] }, { "title": "Available size", "subtitle": "The available capacity size of each TiKV instance", "expressions": [ "sum(( tikv_store_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"} )) by (instance)" ] }, { "title": "Capacity size", "subtitle": "The capacity size per TiKV instance", "expressions": [ "sum(( tikv_store_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"} )) by (instance)" ] }, { "title": "CPU", "subtitle": "The CPU usage of each TiKV instance", "expressions": [ "sum(rate( process_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"} [$__rate_interval] )) by (instance)", "(( tikv_server_cpu_cores_quota {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"} ))" ] }, { "title": "Memory", "subtitle": "The memory usage per TiKV instance", "expressions": [ "sum(( process_resident_memory_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"} )) by (instance)", "(( tikv_server_memory_quota_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"} ))" ] }, { "title": "IO utilization", "subtitle": "The I/O utilization per TiKV instance", "expressions": [ "sum(rate( node_disk_io_time_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "MBps", "subtitle": "The total bytes of read and write in each TiKV instance", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_in_memory_engine_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"} [$__rate_interval] )) by (instance)" ] }, { "title": "QPS", "subtitle": "The number of leaders on each TiKV instance", "expressions": [ "sum(rate( tikv_grpc_msg_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Errps", "subtitle": "The total number of the gRPC message failures", "expressions": [ "sum(rate( tikv_grpc_msg_fail_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"} [$__rate_interval] )) by (instance)", "sum(delta( tikv_pd_heartbeat_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"} [$__rate_interval] )) by (instance) < 1", "sum(rate( tikv_critical_error_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Leader", "subtitle": "The number of leaders on each TiKV instance", "expressions": [ "sum(( tikv_raftstore_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"} )) by (instance)" ] }, { "title": "Region", "subtitle": "The number of Regions and Buckets on each TiKV instance", "expressions": [ "sum(( tikv_raftstore_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"} )) by (instance)" ] }, { "title": "Uptime", "subtitle": "TiKV uptime since the last restart", "expressions": [ "(time() - (( process_start_time_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"} )) )" ] } ] }, { "title": "Errors", "panels": [ { "title": "Critical error", "subtitle": null, "expressions": [ "sum(rate( tikv_critical_error_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Server is busy", "subtitle": "Indicates occurrences of events that make the TiKV instance unavailable\ntemporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor\nFull", "expressions": [ "sum(rate( tikv_scheduler_too_busy_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_channel_full_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)", "sum(rate( tikv_coprocessor_request_error {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"} [$__rate_interval] )) by (instance)", "avg(( tikv_engine_write_stall {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"} )) by (instance, db)", "sum(rate( tikv_raftstore_store_write_msg_block_wait_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)", "sum(( tikv_raftstore_process_busy {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, type)" ] }, { "title": "Server report failures", "subtitle": "The total number of reporting failure messages", "expressions": [ "sum(rate( tikv_server_report_failure_msg_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, instance, store_id)" ] }, { "title": "Raftstore error", "subtitle": "The number of different raftstore errors on each TiKV instance", "expressions": [ "sum(rate( tikv_storage_engine_async_request_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"} [$__rate_interval] )) by (instance, status)" ] }, { "title": "Scheduler error", "subtitle": "The number of scheduler errors per type on each TiKV instance", "expressions": [ "sum(rate( tikv_scheduler_stage_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"} [$__rate_interval] )) by (instance, stage)" ] }, { "title": "Coprocessor error", "subtitle": "The number of different coprocessor errors on each TiKV instance", "expressions": [ "sum(rate( tikv_coprocessor_request_error {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, reason)" ] }, { "title": "gRPC message error", "subtitle": "The number of gRPC message errors per type on each TiKV instance", "expressions": [ "sum(rate( tikv_grpc_msg_fail_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Leader drop", "subtitle": "The count of dropped leaders per TiKV instance", "expressions": [ "sum(delta( tikv_raftstore_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Leader missing", "subtitle": "The count of missing leaders per TiKV instance", "expressions": [ "sum(( tikv_raftstore_leader_missing {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Damaged files", "subtitle": "RocksDB damaged SST files", "expressions": [ "(( tikv_rocksdb_damaged_files {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))", "(( tikv_rocksdb_damaged_files_deleted {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Log Replication Rejected", "subtitle": "The count of Log Replication Reject caused by follower memory insufficient", "expressions": [ "sum(rate( tikv_server_raft_append_rejects {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] } ] }, { "title": "Server", "panels": [ { "title": "CF size", "subtitle": "The size of each column family", "expressions": [ "sum(( tikv_engine_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Channel full", "subtitle": "The total number of channel full errors on each TiKV instance", "expressions": [ "sum(rate( tikv_channel_full_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Active written leaders", "subtitle": "The number of leaders being written on each TiKV instance", "expressions": [ "sum(rate( tikv_region_written_keys_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Approximate region size", "subtitle": null, "expressions": [ "sum(increase( tikv_raftstore_region_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Approximate region size", "subtitle": "The approximate Region size", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_region_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_region_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_region_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_region_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Region written bytes", "subtitle": null, "expressions": [ "sum(increase( tikv_region_written_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region average written bytes", "subtitle": "The average rate of writing bytes to Regions per TiKV instance", "expressions": [ "(sum(rate( tikv_region_written_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_region_written_bytes_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Region written keys", "subtitle": null, "expressions": [ "sum(increase( tikv_region_written_keys_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region average written keys", "subtitle": "The average rate of written keys to Regions per TiKV instance", "expressions": [ "(sum(rate( tikv_region_written_keys_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_region_written_keys_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Clear overlap region duration", "subtitle": "Bucketed histogram of clear overlap region duration.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_clear_overlap_region_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_clear_overlap_region_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_clear_overlap_region_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) / sum(rate( tikv_raftstore_clear_overlap_region_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) )" ] }, { "title": "Written key size", "subtitle": "The key size for every put of apply worker", "expressions": [ "sum(increase( tikv_raftstore_apply_key_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Written value size for every put of apply worker", "subtitle": null, "expressions": [ "sum(increase( tikv_raftstore_apply_value_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Hibernate Peers", "subtitle": "The number of peers in hibernated state", "expressions": [ "sum(( tikv_raftstore_hibernated_peer_state {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, state)" ] }, { "title": "Memory trace", "subtitle": null, "expressions": [ "(( tikv_server_mem_trace_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"} ))", "(( raft_engine_memory_usage {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Raft Entry Cache Evicts", "subtitle": null, "expressions": [ "sum(rate( tikv_raft_entries_evict_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Resolve address duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_server_address_resolve_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "99% Thread Pool Schedule Wait Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_yatp_pool_schedule_wait_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, priority, le, $additional_groupby) ))" ] }, { "title": "Average Thread Pool Schedule Wait Duration", "subtitle": "The average rate of written keys to Regions per TiKV instance", "expressions": [ "(sum(rate( tikv_yatp_pool_schedule_wait_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, priority, $additional_groupby) / sum(rate( tikv_yatp_pool_schedule_wait_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, priority, $additional_groupby) )" ] }, { "title": "Disk IO time per second", "subtitle": null, "expressions": [ "sum(rate( tikv_storage_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"} [$__rate_interval] )) by (req, $additional_groupby)", "sum(rate( tikv_coprocessor_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"} [$__rate_interval] )) by (req, $additional_groupby)" ] }, { "title": "Disk IO bytes per second", "subtitle": null, "expressions": [ "sum(rate( tikv_storage_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"} [$__rate_interval] )) by (req, $additional_groupby)", "sum(rate( tikv_coprocessor_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"} [$__rate_interval] )) by (req, $additional_groupby)" ] } ] }, { "title": "gRPC", "panels": [ { "title": "gRPC message count", "subtitle": "The count of different kinds of gRPC message", "expressions": [ "sum(rate( tikv_grpc_msg_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "gRPC message failed", "subtitle": "The count of different kinds of gRPC message which is failed", "expressions": [ "sum(rate( tikv_grpc_msg_fail_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "99% gRPC message duration", "subtitle": "The 99% percentile of execution time of gRPC message", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_grpc_msg_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))" ] }, { "title": "Average gRPC message duration", "subtitle": "The average execution time of gRPC message", "expressions": [ "(sum(rate( tikv_grpc_msg_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) / sum(rate( tikv_grpc_msg_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) )" ] }, { "title": "gRPC batch commands wait duration", "subtitle": "The 99.99% wait time of gRPC batch commands", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_grpc_batch_commands_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "gRPC batch size", "subtitle": "The 99% percentile of execution time of gRPC message", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_server_grpc_req_batch_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_server_grpc_resp_batch_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_server_grpc_req_batch_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_server_grpc_req_batch_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "(sum(rate( tikv_server_grpc_resp_batch_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_server_grpc_resp_batch_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "histogram_quantile(0.99,( sum(rate( tikv_server_request_batch_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_server_request_batch_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_server_request_batch_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "raft message batch size", "subtitle": null, "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_server_raft_message_batch_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_server_raft_message_batch_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_server_raft_message_batch_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "gRPC request sources QPS", "subtitle": "The QPS of different sources of gRPC request", "expressions": [ "sum(rate( tikv_grpc_request_source_counter_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (source, $additional_groupby)" ] }, { "title": "gRPC request sources duration", "subtitle": "The duration of different sources of gRPC request", "expressions": [ "sum(rate( tikv_grpc_request_source_duration_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (source, $additional_groupby)" ] }, { "title": "gRPC resource group QPS", "subtitle": "The QPS of different resource groups of gRPC request", "expressions": [ "sum(rate( tikv_grpc_resource_group_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, $additional_groupby)" ] } ] }, { "title": "Storage", "panels": [ { "title": "Storage command total", "subtitle": "The total count of different kinds of commands received", "expressions": [ "sum(rate( tikv_storage_command_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Storage async request error", "subtitle": "The total number of engine asynchronous request errors", "expressions": [ "sum(rate( tikv_storage_engine_async_request_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"} [$__rate_interval] )) by (status, $additional_groupby)" ] }, { "title": "Storage async write duration", "subtitle": "The time consumed by processing asynchronous write requests", "expressions": [ "sum(increase( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by (le)" ] }, { "title": "Storage async write duration", "subtitle": "The storage async write duration", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Storage async snapshot duration", "subtitle": "The time consumed by processing asynchronous snapshot requests", "expressions": [ "sum(increase( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by (le)" ] }, { "title": "Storage async snapshot duration", "subtitle": "The storage async snapshot duration", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Storage async snapshot duration (pure local read)", "subtitle": "The storage async snapshot duration without the involving of raftstore", "expressions": [ "sum(increase( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"} [$__rate_interval] )) by (le)" ] }, { "title": "Storage async snapshot duration (pure local read)", "subtitle": "The storage async snapshot duration without the involving of raftstore", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Read index propose wait duration", "subtitle": "Read index propose wait duration associated with async snapshot", "expressions": [ "sum(increase( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"} [$__rate_interval] )) by (le)" ] }, { "title": "Read index propose wait duration", "subtitle": "Read index propose wait duration associated with async snapshot", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Read index confirm duration", "subtitle": "Read index confirm duration associated with async snapshot", "expressions": [ "sum(increase( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"} [$__rate_interval] )) by (le)" ] }, { "title": "Read index confirm duration", "subtitle": "Read index confirm duration associated with async snapshot", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Process Stat Cpu Usage", "subtitle": "CPU usage measured over a 30 second window", "expressions": [ "sum(( tikv_storage_process_stat_cpu_usage {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, $additional_groupby)" ] }, { "title": "Full compaction duration seconds", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_full_compact_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_full_compact_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_full_compact_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_full_compact_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Full compaction pause duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_full_compact_pause_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_full_compact_pause_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_full_compact_pause_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_full_compact_pause_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Full compaction per-increment duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_full_compact_increment_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_full_compact_increment_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_full_compact_increment_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_full_compact_increment_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Concurrency manager max-ts", "subtitle": "The max_ts in the concurrency manager", "expressions": [ "tikv_concurrency_manager_max_ts_limit", "tikv_concurrency_manager_max_ts" ] } ] }, { "title": "Local Reader", "panels": [ { "title": "Local reader requests", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_local_read_reject_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, reason)", "sum(rate( tikv_raftstore_local_read_executed_requests {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_raftstore_local_read_executed_stale_read_requests {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] } ] }, { "title": "Thread CPU", "panels": [ { "title": "Raft store CPU", "subtitle": "The CPU utilization of raftstore thread", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Async apply CPU", "subtitle": "The CPU utilization of async apply", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Store writer CPU", "subtitle": "The CPU utilization of store writer thread", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "gRPC poll CPU", "subtitle": "The CPU utilization of gRPC", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Scheduler worker CPU", "subtitle": "The CPU utilization of scheduler worker", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Unified read pool CPU", "subtitle": "The CPU utilization of the unified read pool", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "RocksDB CPU", "subtitle": "The CPU utilization of RocksDB", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "GC worker CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Region worker CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"region_worker.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Snap generator CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"snap_generator.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Background worker CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raftlog fetch worker CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Import CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Backup CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "CDC worker CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "TSO worker CPU", "subtitle": "The CPU utilization of raftstore thread", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Storage read pool CPU", "subtitle": "The CPU utilization of storage read pool", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Coprocessor read pool CPU", "subtitle": "The CPU utilization of coprocessor read pool", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "IME CPU", "subtitle": "The CPU utilization of IME threads", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Busy Threads (>80%)", "subtitle": null, "expressions": [ "topk(20,( sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name!~\"rocksdb.*\"} [$__rate_interval] )) by (instance, name) > 0.8 ))" ] } ] }, { "title": "IO Breakdown", "panels": [ { "title": "Write IO bytes", "subtitle": "The throughput of disk write per IO type", "expressions": [ "sum(rate( tikv_io_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"} [$__rate_interval] )) by (type, $additional_groupby)", "sum(rate( tikv_io_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Read IO bytes", "subtitle": "The throughput of disk read per IO type", "expressions": [ "sum(rate( tikv_io_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"} [$__rate_interval] )) by (type, $additional_groupby)", "sum(rate( tikv_io_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "IO threshold", "subtitle": "The threshold of disk IOs per priority", "expressions": [ "avg(( tikv_rate_limiter_max_bytes_per_sec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Rate Limiter Request Wait Duration", "subtitle": "IO rate limiter request wait duration.", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_rate_limiter_request_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "(sum(rate( tikv_rate_limiter_request_wait_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_rate_limiter_request_wait_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] } ] }, { "title": "Raft Waterfall", "panels": [ { "title": "Storage async write duration", "subtitle": "The time consumed by processing asynchronous write requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_engine_async_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_storage_engine_async_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_storage_engine_async_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store duration", "subtitle": "The store time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Apply duration", "subtitle": "The apply time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_apply_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_apply_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_apply_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_apply_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store propose wait duration", "subtitle": "The propose wait time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_request_wait_time_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_request_wait_time_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_request_wait_time_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store batch wait duration", "subtitle": "The batch wait time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_batch_wait_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_batch_wait_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_batch_wait_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store send to write queue duration", "subtitle": "The send-to-write-queue time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_send_to_queue_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_send_to_queue_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store send proposal duration", "subtitle": "The send raft message of the proposal duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_send_proposal_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_send_proposal_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_send_proposal_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store write kv db end duration", "subtitle": "The write kv db end duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store before write duration", "subtitle": "The before write time duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_before_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_before_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_before_write_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_before_write_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_before_write_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store persist duration", "subtitle": "The persist duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_persist_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_persist_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_persist_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_persist_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_persist_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store write end duration", "subtitle": "The write end duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_write_end_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_write_end_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_write_end_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_write_end_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_write_end_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store commit but not persist duration", "subtitle": "The commit but not persist duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Store commit and persist duration", "subtitle": "The commit and persist duration of each request", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_wf_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_wf_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_store_wf_commit_log_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_raftstore_store_wf_commit_log_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_raftstore_store_wf_commit_log_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] } ] }, { "title": "Raftstore IO", "panels": [ { "title": "Peer destroy kv write", "subtitle": "The time taken by Raftstore to complete RocksDB write operation when destroying a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_kv_write\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Peer destroy kv write", "subtitle": "The time taken by Raftstore to complete RocksDB write operation when destroying a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_kv_write\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_kv_write\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Peer destroy raft write", "subtitle": "The time taken by Raftstore to complete RaftEngine write operation when destroying a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_raft_write\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Peer destroy raft write", "subtitle": "The time taken by Raftstore to complete RaftEngine write operation when destroying a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_raft_write\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_destroy_raft_write\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Init raft state", "subtitle": "The time taken by Raftstore to complete initializing raft state when creating a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_raft_state\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Init raft state", "subtitle": "The time taken by Raftstore to complete initializing raft state when creating a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_raft_state\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_raft_state\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Init apply state", "subtitle": "The time taken by Raftstore to complete initializing apply state when creating a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_apply_state\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Init apply state", "subtitle": "The time taken by Raftstore to complete initializing apply state when creating a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_apply_state\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"init_apply_state\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Entry storage create", "subtitle": "The time taken by Raftstore to complete RaftEngine read operation when creating a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"entry_storage_create\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Entry storage create", "subtitle": "The time taken by Raftstore to complete RaftEngine read operation when creating a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"entry_storage_create\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"entry_storage_create\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Store check msg", "subtitle": "The time taken by Raftstore to check region state when TiKV receives an message to non-exist regions", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"store_check_msg\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Store check msg", "subtitle": "The time taken by Raftstore to check region state when TiKV receives an message to non-exist regions", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"store_check_msg\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"store_check_msg\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Peer check merge target stale", "subtitle": "The time taken by Raftstore to check stale merged regions", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_check_merge_target_stale\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Peer check merge target stale", "subtitle": "The time taken by Raftstore to check stale merged regions", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_check_merge_target_stale\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_check_merge_target_stale\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Peer maybe create", "subtitle": "The time taken by Raftstore to complete RocksDB and RaftEngine read operation when creating a peer", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_maybe_create\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Peer maybe create", "subtitle": "The time taken by Raftstore to complete RocksDB and RaftEngine read operation when creating a peer", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_maybe_create\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_maybe_create\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Peer snapshot read", "subtitle": "The time taken by Raftstore to complete a read requests", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_snapshot_read\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Peer snapshot read", "subtitle": "The time taken by Raftstore to complete a read requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_snapshot_read\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"peer_snapshot_read\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "V2 compatible learner", "subtitle": "The time taken by Raftstore to handle raftstore v2 compatibility checking", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"v2_compatible_learner\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% V2 compatible learner", "subtitle": "The time taken by Raftstore to handle raftstore v2 compatibility checking", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"v2_compatible_learner\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"v2_compatible_learner\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Raft term", "subtitle": "The time taken by Raftstore to get terms of raft logs by reading RaftEngine", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_term\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Raft term", "subtitle": "The time taken by Raftstore to get terms of raft logs by reading RaftEngine", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_term\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_term\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Raft fetch log", "subtitle": "The time taken by Raftstore to fetch raft logs by reading RaftEngine", "expressions": [ "sum(increase( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_fetch_log\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Raft fetch log", "subtitle": "The time taken by Raftstore to fetch raft logs by reading RaftEngine", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_fetch_log\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_io_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason=\"raft_fetch_log\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] } ] }, { "title": "Raft IO", "panels": [ { "title": "Process ready duration", "subtitle": "The time consumed for peer processes to be ready in Raft", "expressions": [ "sum(increase( tikv_raftstore_raft_process_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Process ready duration per server", "subtitle": "The time consumed for peer processes to be ready in Raft", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_raft_process_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_raft_process_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Store write loop duration", "subtitle": "The time duration of store write loop when store-io-pool-size is not zero.", "expressions": [ "sum(increase( tikv_raftstore_store_write_loop_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Store write loop duration per server", "subtitle": "The time duration of store write loop on each TiKV instance when store-io-pool-size is not zero.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_write_loop_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_write_loop_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Append log duration", "subtitle": "The time consumed when Raft appends log", "expressions": [ "sum(increase( tikv_raftstore_append_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Append log duration per server", "subtitle": "The time consumed when Raft appends log on each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_append_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_append_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Commit log duration", "subtitle": "The time consumed when Raft commits log", "expressions": [ "sum(increase( tikv_raftstore_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Commit log duration per server", "subtitle": "The time consumed when Raft commits log on each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_commit_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Apply log duration", "subtitle": "The time consumed when Raft applies log", "expressions": [ "sum(increase( tikv_raftstore_apply_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Apply log duration per server", "subtitle": "The time consumed for Raft to apply logs per TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_apply_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Raft Client Wait Connection Ready Duration", "subtitle": "The time consumed for Raft Client wait connection ready", "expressions": [ "sum(increase( tikv_server_raft_client_wait_ready_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Raft Client Wait Connection Ready Duration", "subtitle": "The time consumed for Raft Client wait connection ready per TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_server_raft_client_wait_ready_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (to, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_server_raft_client_wait_ready_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (to, le, $additional_groupby) ))", "(sum(rate( tikv_server_raft_client_wait_ready_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (to, $additional_groupby) / sum(rate( tikv_server_raft_client_wait_ready_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (to, $additional_groupby) )" ] }, { "title": "Store io task reschedule", "subtitle": "The throughput of disk write per IO type", "expressions": [ "sum(( tikv_raftstore_io_reschedule_region_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_raftstore_io_reschedule_pending_tasks_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "99% Write task block duration per server", "subtitle": "The time consumed when store write task block on each TiKV instance", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] } ] }, { "title": "Raft Propose", "panels": [ { "title": "Raft proposals per ready", "subtitle": "The proposal count of a Regions in a tick", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_proposal_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "Raft read/write proposals", "subtitle": "The number of proposals per type", "expressions": [ "sum(rate( tikv_raftstore_proposal_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Raft read proposals per server", "subtitle": "The number of read proposals which are made by each TiKV instance", "expressions": [ "sum(rate( tikv_raftstore_proposal_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft write proposals per server", "subtitle": "The number of write proposals which are made by each TiKV instance", "expressions": [ "sum(rate( tikv_raftstore_proposal_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Propose wait duration", "subtitle": "The wait time of each proposal", "expressions": [ "sum(increase( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Propose wait duration per server", "subtitle": "The wait time of each proposal in each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_request_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Store write wait duration", "subtitle": "The wait time of each store write task", "expressions": [ "sum(increase( tikv_raftstore_store_write_task_wait_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Store write wait duration per server", "subtitle": "The wait time of each store write task in each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_store_write_task_wait_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_write_task_wait_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Apply wait duration", "subtitle": "The wait time of each apply task", "expressions": [ "sum(increase( tikv_raftstore_apply_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% Apply wait duration per server", "subtitle": "The wait time of each apply task in each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_apply_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_wait_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Store write handle msg duration", "subtitle": "The handle duration of each store write task msg", "expressions": [ "sum(increase( tikv_raftstore_store_write_handle_msg_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Store write trigger size", "subtitle": "The distribution of write trigger size", "expressions": [ "sum(increase( tikv_raftstore_store_write_trigger_wb_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Raft propose speed", "subtitle": "The rate at which peers propose logs", "expressions": [ "sum(rate( tikv_raftstore_propose_log_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Perf Context duration", "subtitle": "The rate at which peers propose logs", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_store_perf_context_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_apply_perf_context_time_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))" ] } ] }, { "title": "Raft Process", "panels": [ { "title": "Ready handled", "subtitle": "The count of different ready type of Raft", "expressions": [ "sum(rate( tikv_raftstore_raft_ready_handled_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)", "sum(rate( tikv_raftstore_raft_process_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Max duration of raft store events", "subtitle": "The max time consumed by raftstore events", "expressions": [ "histogram_quantile(0.999999,( sum(rate( tikv_raftstore_event_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.999999,( sum(rate( tikv_broadcast_normal_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Replica read lock checking duration", "subtitle": "Replica read lock checking duration", "expressions": [ "sum(increase( tikv_replica_read_lock_check_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Fsm reschedule ops", "subtitle": "The number of fsm reschedule ops", "expressions": [ "sum(rate( tikv_batch_system_fsm_reschedule_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type)" ] }, { "title": "Store fsm schedule wait duration", "subtitle": "Duration of store fsm waiting to be polled", "expressions": [ "sum(increase( tikv_batch_system_fsm_schedule_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply fsm schedule wait duration", "subtitle": "Duration of apply fsm waiting to be polled.e", "expressions": [ "sum(increase( tikv_batch_system_fsm_schedule_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Store fsm poll duration", "subtitle": "Total time for an store FSM to finish processing all messages, potentially over multiple polling rounds.", "expressions": [ "sum(increase( tikv_batch_system_fsm_poll_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply fsm poll duration", "subtitle": "Total time for an apply FSM to finish processing all messages, potentially over multiple polling rounds", "expressions": [ "sum(increase( tikv_batch_system_fsm_poll_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Store fsm poll round", "subtitle": "Number of polling rounds for an store FSM to finish processing all messages", "expressions": [ "sum(increase( tikv_batch_system_fsm_poll_rounds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply fsm poll round", "subtitle": "Number of polling rounds for an apply FSM to finish processing all messages", "expressions": [ "sum(increase( tikv_batch_system_fsm_poll_rounds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Store fsm count per poll", "subtitle": "Number of store fsm polled in one poll", "expressions": [ "sum(increase( tikv_batch_system_fsm_count_per_poll_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply fsm count per poll", "subtitle": "Number of apply fsm polled in one poll", "expressions": [ "sum(increase( tikv_batch_system_fsm_count_per_poll_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Peer msg length distribution", "subtitle": "The length of peer msgs for each round handling", "expressions": [ "sum(increase( tikv_raftstore_peer_msg_len_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply msg length distribution", "subtitle": "The length of apply msgs for each round handling", "expressions": [ "sum(increase( tikv_raftstore_apply_msg_len_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Enable apply unpersisted log region count", "subtitle": "The number of regions that enable apply unpersisted raft log", "expressions": [ "(( tikv_raft_enable_unpersisted_apply_regions {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Apply ahead of persistence raft log count", "subtitle": "The number of raft logs between apply and persisted index", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raft_apply_ahead_of_persist_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "histogram_quantile(1,( sum(rate( tikv_raft_apply_ahead_of_persist_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] } ] }, { "title": "Raft Message", "panels": [ { "title": "Sent messages per server", "subtitle": "The number of Raft messages sent by each TiKV instance", "expressions": [ "sum(rate( tikv_raftstore_raft_sent_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Flush messages per server", "subtitle": "The number of Raft messages flushed by each TiKV instance", "expressions": [ "sum(rate( tikv_server_raft_message_flush_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, reason)" ] }, { "title": "Receive messages per server", "subtitle": "The number of Raft messages received by each TiKV instance", "expressions": [ "sum(rate( tikv_server_raft_message_recv_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Messages", "subtitle": "The number of different types of Raft messages that are sent", "expressions": [ "sum(rate( tikv_raftstore_raft_sent_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"accept\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Vote", "subtitle": "The total number of vote messages that are sent in Raft", "expressions": [ "sum(rate( tikv_raftstore_raft_sent_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft dropped messages", "subtitle": "The number of dropped Raft messages per type", "expressions": [ "sum(rate( tikv_raftstore_raft_dropped_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)", "sum(rate( tikv_raftstore_raft_sent_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"drop\"} [$__rate_interval] )) by (type, $additional_groupby)" ] } ] }, { "title": "Raft Admin", "panels": [ { "title": "Admin proposals", "subtitle": "The number of admin proposals", "expressions": [ "sum(rate( tikv_raftstore_proposal_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Admin apply", "subtitle": "The number of the processed apply command", "expressions": [ "sum(rate( tikv_raftstore_admin_cmd_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Check split", "subtitle": "The number of raftstore split checks", "expressions": [ "sum(rate( tikv_raftstore_check_split_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "99.99% Check split duration", "subtitle": "The time consumed when running split check in .9999", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_check_split_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "Load base split event", "subtitle": null, "expressions": [ "sum(delta( tikv_load_base_split_event {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, $additional_groupby)" ] }, { "title": "Load base split duration", "subtitle": null, "expressions": [ "histogram_quantile(0.8,( sum(rate( tikv_load_base_split_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "histogram_quantile(0.99,( sum(rate( tikv_load_base_split_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "(sum(rate( tikv_load_base_split_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_load_base_split_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Peer in Flashback State", "subtitle": null, "expressions": [ "sum(( tikv_raftstore_peer_in_flashback_state {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] } ] }, { "title": "Raft Log", "panels": [ { "title": "Raft log GC write duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_raft_log_gc_write_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "(sum(rate( tikv_raftstore_raft_log_gc_write_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_raftstore_raft_log_gc_write_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Raft log GC kv sync duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_raft_log_kv_sync_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "(sum(rate( tikv_raftstore_raft_log_kv_sync_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_raftstore_raft_log_kv_sync_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Raft log GC write operations", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_raft_log_gc_write_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft log GC seek operations", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_raft_log_gc_seek_operations_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft log lag", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_log_lag_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft log gc skipped", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_raft_log_gc_skipped {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, reason)" ] }, { "title": "Raft log GC failed", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_raft_log_gc_failed {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Raft log fetch", "subtitle": null, "expressions": [ "sum(rate( tikv_raftstore_entry_fetches {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Raft log async fetch task duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_raftstore_entry_fetches_task_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_raftstore_entry_fetches_task_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_raftstore_entry_fetches_task_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) )", "sum(( tikv_worker_pending_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"} )) by (instance, $additional_groupby)" ] } ] }, { "title": "Raft Engine", "panels": [ { "title": "Operation", "subtitle": "The count of operations per second", "expressions": [ "sum(rate( raft_engine_write_apply_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( raft_engine_read_entry_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( raft_engine_read_message_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Write Duration", "subtitle": "The time used in write operation", "expressions": [ "histogram_quantile(0.9999,( sum(rate( raft_engine_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( raft_engine_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( raft_engine_write_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( raft_engine_write_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Flow", "subtitle": "The I/O flow rate", "expressions": [ "sum(rate( raft_engine_write_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( raft_engine_background_rewrite_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Write Duration Breakdown (99%)", "subtitle": "99% duration breakdown of write operation", "expressions": [ "histogram_quantile(0.99,( sum(rate( raft_engine_write_preprocess_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( raft_engine_write_leader_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( raft_engine_write_apply_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Bytes / Written", "subtitle": "The bytes per write", "expressions": [ "histogram_quantile(0.9999,( sum(rate( raft_engine_write_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( raft_engine_write_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( raft_engine_write_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( raft_engine_write_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "WAL Duration Breakdown (999%)", "subtitle": "999% duration breakdown of WAL write operation", "expressions": [ "histogram_quantile(0.999,( sum(rate( raft_engine_write_leader_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.999,( sum(rate( raft_engine_sync_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.999,( sum(rate( raft_engine_allocate_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.999,( sum(rate( raft_engine_rotate_log_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "File Count", "subtitle": "The average number of files", "expressions": [ "avg(( raft_engine_log_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)", "avg(( raft_engine_swap_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by ($additional_groupby)", "avg(( raft_engine_recycled_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Other Durations (99%)", "subtitle": "The 99% duration of operations other than write", "expressions": [ "histogram_quantile(0.999,( sum(rate( raft_engine_read_entry_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.999,( sum(rate( raft_engine_read_message_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.999,( sum(rate( raft_engine_purge_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Entry Count", "subtitle": "The average number of log entries", "expressions": [ "avg(( raft_engine_log_entry_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Write Compression Ratio", "subtitle": "The compression ratio per write", "expressions": [ "histogram_quantile(0.9999,( sum(rate( raft_engine_write_compression_ratio_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( raft_engine_write_compression_ratio_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( raft_engine_write_compression_ratio_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( raft_engine_write_compression_ratio_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( raft_engine_write_compression_ratio_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] } ] }, { "title": "RocksDB - $db", "panels": [ { "title": "Get operations", "subtitle": "The count of get operations", "expressions": [ "sum(rate( tikv_engine_memtable_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_get_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_get_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_get_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Get duration", "subtitle": "The time consumed when executing get operations", "expressions": [ "max(( tikv_engine_get_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_get_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_get_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_get_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"} )) by ($additional_groupby)" ] }, { "title": "Seek operations", "subtitle": "The count of seek operations", "expressions": [ "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Seek duration", "subtitle": "The time consumed when executing seek operation", "expressions": [ "max(( tikv_engine_seek_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_seek_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_seek_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_seek_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"} )) by ($additional_groupby)" ] }, { "title": "Write operations", "subtitle": "The count of write operations", "expressions": [ "sum(rate( tikv_engine_write_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_write_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_write_served {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Write duration", "subtitle": "The time consumed when executing write operation", "expressions": [ "max(( tikv_engine_write_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"} )) by ($additional_groupby)" ] }, { "title": "WAL sync operations", "subtitle": "The count of WAL sync operations", "expressions": [ "sum(rate( tikv_engine_wal_file_synced {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Write WAL duration", "subtitle": "The time consumed when executing write wal operation", "expressions": [ "max(( tikv_engine_write_wal_time_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_wal_time_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_wal_time_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_wal_time_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"} )) by ($additional_groupby)" ] }, { "title": "Compaction operations", "subtitle": "The count of compaction and flush operations", "expressions": [ "sum(rate( tikv_engine_event_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "WAL sync duration", "subtitle": "The time consumed when executing WAL sync operation", "expressions": [ "max(( tikv_engine_wal_file_sync_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_wal_file_sync_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_wal_file_sync_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_wal_file_sync_micro_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"} )) by ($additional_groupby)" ] }, { "title": "Compaction guard actions", "subtitle": "Compaction guard actions", "expressions": [ "sum(rate( tikv_raftstore_compaction_guard_action_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"} [$__rate_interval] )) by (cf, type, $additional_groupby)" ] }, { "title": "Compaction duration", "subtitle": "The time consumed when executing the compaction and flush operations", "expressions": [ "max(( tikv_engine_compaction_time {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_compaction_time {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_compaction_time {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_compaction_time {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"} )) by ($additional_groupby)" ] }, { "title": "Compaction Job Size(files)", "subtitle": "How many sst files are compacted in a compaction job", "expressions": [ "max(( tikv_engine_num_files_in_single_compaction {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_num_files_in_single_compaction {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_num_files_in_single_compaction {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_num_files_in_single_compaction {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_average\"} )) by ($additional_groupby)" ] }, { "title": "SST read duration", "subtitle": "The time consumed when reading SST files", "expressions": [ "max(( tikv_engine_sst_read_micros {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_sst_read_micros {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_sst_read_micros {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_sst_read_micros {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"} )) by ($additional_groupby)" ] }, { "title": "Compaction reason", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_compaction_reason {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by (cf, reason, $additional_groupby)" ] }, { "title": "Block cache size", "subtitle": "The block cache size. Broken down by column family if shared block cache is disabled.", "expressions": [ "topk(20,( avg(( tikv_engine_block_cache_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} )) by (cf, instance) ))" ] }, { "title": "Memtable hit", "subtitle": "The hit rate of memtable", "expressions": [ "(sum(rate( tikv_engine_memtable_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_memtable_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_memtable_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"} [$__rate_interval] )) by ($additional_groupby) ))" ] }, { "title": "Block cache flow", "subtitle": "The flow of different kinds of block cache operations", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Block cache hit", "subtitle": "The hit rate of block cache", "expressions": [ "(sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"} [$__rate_interval] )) by ($additional_groupby) ))", "(sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"} [$__rate_interval] )) by ($additional_groupby) ))", "(sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"} [$__rate_interval] )) by ($additional_groupby) ))", "(sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"} [$__rate_interval] )) by ($additional_groupby) ))", "((sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"} [$__rate_interval] )) by ($additional_groupby) ) / ((sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"} [$__rate_interval] )) by ($additional_groupby) ) + (sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filter_match\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_bloom_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filter_match\"} [$__rate_interval] )) by ($additional_groupby) )))" ] }, { "title": "Keys flow", "subtitle": "The flow of different kinds of operations on keys", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_compaction_num_corrupt_keys {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Block cache operations", "subtitle": "The count of different kinds of block cache operations", "expressions": [ "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Read flow", "subtitle": "The flow rate of read operations per type", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Total keys", "subtitle": "The count of keys in each column family", "expressions": [ "sum(( tikv_engine_estimate_num_keys {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} )) by (cf, $additional_groupby)" ] }, { "title": "Write flow", "subtitle": "The flow of different kinds of write operations", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Bytes / Read", "subtitle": "The bytes per read", "expressions": [ "max(( tikv_engine_bytes_per_read {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_read {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_read {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_read {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"} )) by ($additional_groupby)" ] }, { "title": "Compaction flow", "subtitle": "The flow rate of compaction operations per type", "expressions": [ "sum(rate( tikv_engine_compaction_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_compaction_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Bytes / Write", "subtitle": "The bytes per write", "expressions": [ "max(( tikv_engine_bytes_per_write {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_write {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_write {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_bytes_per_write {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"} )) by ($additional_groupby)" ] }, { "title": "Read amplification", "subtitle": "The read amplification per TiKV instance", "expressions": [ "(sum(rate( tikv_engine_read_amp_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_engine_read_amp_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "Compaction pending bytes", "subtitle": "The pending bytes to be compacted", "expressions": [ "sum(( tikv_engine_pending_compaction_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} )) by (cf, $additional_groupby)" ] }, { "title": "Number of snapshots", "subtitle": "The number of snapshot of each TiKV instance", "expressions": [ "(( tikv_engine_num_snapshots {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} ))" ] }, { "title": "Compression ratio", "subtitle": "The compression ratio of each level", "expressions": [ "avg(( tikv_engine_compression_ratio {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} )) by (cf, level, $additional_groupby)" ] }, { "title": "Number files at each level", "subtitle": "The number of SST files for different column families in each level", "expressions": [ "avg(( tikv_engine_num_files_at_level {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} )) by (cf, level, $additional_groupby)" ] }, { "title": "Oldest snapshots duration", "subtitle": "The time that the oldest unreleased snapshot survivals", "expressions": [ "(( tikv_engine_oldest_snapshot_duration {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} ))" ] }, { "title": "Ingestion picked level", "subtitle": "The level that the external file ingests into", "expressions": [ "sum(increase( tikv_engine_ingestion_picked_level_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by (le)" ] }, { "title": "Ingest SST duration seconds", "subtitle": "Bucketed histogram of ingest external SST files duration.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_ingest_external_file_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (cf, type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_ingest_external_file_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (cf, type, le, $additional_groupby) ))", "(sum(rate( tikv_storage_ingest_external_file_duration_secs_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (cf, type, $additional_groupby) / sum(rate( tikv_storage_ingest_external_file_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (cf, type, $additional_groupby) )" ] }, { "title": "Ingest SST allow_write", "subtitle": null, "expressions": [ "sum(rate( tikv_storage_ingest_external_file_allow_write_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Write Stall Reason", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_write_stall_reason {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Write stall duration", "subtitle": "The time which is caused by write stall", "expressions": [ "max(( tikv_engine_write_stall {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_stall {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_stall {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_write_stall {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"} )) by ($additional_groupby)" ] }, { "title": "Stall conditions changed of each CF", "subtitle": "Stall conditions changed of each column family", "expressions": [ "(( tikv_engine_stall_conditions_changed {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"} ))" ] }, { "title": "Memtable size", "subtitle": "The memtable size of each column family", "expressions": [ "avg(( tikv_engine_memory_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"} )) by (cf, $additional_groupby)" ] } ] }, { "title": "Titan - $titan_db", "panels": [ { "title": "Blob file count", "subtitle": null, "expressions": [ "sum(( tikv_engine_titandb_num_live_blob_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by ($additional_groupby)", "sum(( tikv_engine_titandb_num_obsolete_blob_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by ($additional_groupby)" ] }, { "title": "Blob file size", "subtitle": null, "expressions": [ "sum(( tikv_engine_titandb_live_blob_file_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by ($additional_groupby)", "sum(( tikv_engine_titandb_obsolete_blob_file_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by ($additional_groupby)" ] }, { "title": "Blob cache size", "subtitle": "The blob cache size.", "expressions": [ "topk(20,( avg(( tikv_engine_blob_cache_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by (cf, instance) ))" ] }, { "title": "Blob cache hit", "subtitle": "The hit rate of block cache", "expressions": [ "(sum(rate( tikv_engine_blob_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"} [$__rate_interval] )) by ($additional_groupby) / (sum(rate( tikv_engine_blob_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"} [$__rate_interval] )) by ($additional_groupby) + sum(rate( tikv_engine_blob_cache_efficiency {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"} [$__rate_interval] )) by ($additional_groupby) ))" ] }, { "title": "Iter touched blob file count", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_iter_touch_blob_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_iter_touch_blob_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_iter_touch_blob_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_iter_touch_blob_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob key size", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_key_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_key_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_key_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_key_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob value size", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_value_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_value_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_value_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_value_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob get operations", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Blob get duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_get_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_get_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_get_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_get_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob file discardable ratio distribution", "subtitle": null, "expressions": [ "sum(( tikv_engine_titandb_blob_file_discardable_ratio {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} )) by (ratio, $additional_groupby)" ] }, { "title": "Blob iter operations", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_blob_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_engine_blob_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Blob seek duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_seek_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_seek_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_seek_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_seek_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob next duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_next_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_next_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_next_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_next_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob prev duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_prev_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_prev_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_prev_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_prev_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob keys flow", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Blob file read duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_file_read_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_read_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_read_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_file_read_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob bytes flow", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Blob file write duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_file_write_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_write_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_write_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_file_write_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob file sync operations", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_file_synced {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Blob GC action", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_gc_action_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Blob file sync duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_file_sync_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_sync_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_file_sync_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_file_sync_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob GC duration", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_gc_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_gc_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"} )) by (type, $additional_groupby)", "avg(( tikv_engine_blob_gc_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"} )) by (type, $additional_groupby)", "max(( tikv_engine_blob_gc_micros_seconds {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"} )) by (type, $additional_groupby)" ] }, { "title": "Blob GC keys flow", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_gc_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Blob GC input file size", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_gc_input_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_gc_input_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_gc_input_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_gc_input_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob GC bytes flow", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_gc_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Blob GC output file size", "subtitle": null, "expressions": [ "avg(( tikv_engine_blob_gc_output_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_gc_output_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"} )) by ($additional_groupby)", "avg(( tikv_engine_blob_gc_output_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"} )) by ($additional_groupby)", "max(( tikv_engine_blob_gc_output_file {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"} )) by ($additional_groupby)" ] }, { "title": "Blob GC file count", "subtitle": null, "expressions": [ "sum(rate( tikv_engine_blob_gc_file_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"} [$__rate_interval] )) by (type, $additional_groupby)" ] } ] }, { "title": "In Memory Engine", "panels": [ { "title": "OPS", "subtitle": "Operation per second for cf", "expressions": [ "sum(rate( tikv_in_memory_engine_kv_operations {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type, $additional_groupby)" ] }, { "title": "Read MBps", "subtitle": "The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)", "expressions": [ "sum(rate( tikv_engine_flow_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_in_memory_engine_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Coprocessor Handle duration", "subtitle": "The time consumed when handling coprocessor requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))" ] }, { "title": "Region Cache Hit", "subtitle": "Count of region cache hit", "expressions": [ "sum(rate( tikv_snapshot_type_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Region Cache Hit Rate", "subtitle": "Region cache hit rate", "expressions": [ "(sum(rate( tikv_snapshot_type_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"} [$__rate_interval] )) by (instance, $additional_groupby) / sum(rate( tikv_snapshot_type_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, $additional_groupby) )" ] }, { "title": "Region Cache Miss Reason", "subtitle": "Reasons for region cache miss", "expressions": [ "sum(rate( tikv_in_memory_engine_snapshot_acquire_failed_reason_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Memory Usage", "subtitle": "The memory usage of the in-memory engine", "expressions": [ "avg(( tikv_in_memory_engine_memory_usage_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Region Count", "subtitle": "The count of different types of region", "expressions": [ "avg(( tikv_in_memory_engine_cache_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, type)" ] }, { "title": "GC Filter", "subtitle": "Rang cache engine garbage collection information", "expressions": [ "sum(rate( tikv_in_memory_engine_gc_filtered {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Region GC Duration", "subtitle": "The handle duration of region gc", "expressions": [ "sum(increase( tikv_in_memory_engine_gc_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region Load Duration", "subtitle": "The handle duration of region load", "expressions": [ "sum(increase( tikv_in_memory_engine_load_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region Load Count", "subtitle": "The count of region loading per seconds", "expressions": [ "sum(delta( tikv_in_memory_engine_load_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Region Eviction Duration", "subtitle": "The handle duration of region eviction", "expressions": [ "sum(increase( tikv_in_memory_engine_eviction_duration_secs_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region Eviction Count", "subtitle": "The count of region eviction per seconds", "expressions": [ "sum(delta( tikv_in_memory_engine_eviction_duration_secs_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Region Warmup Count", "subtitle": "The count of region warmup per seconds", "expressions": [ "sum(rate( tikv_in_memory_engine_transfer_leader_warmup_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Write duration", "subtitle": "The time consumed of write in region cache engine", "expressions": [ "sum(increase( tikv_in_memory_engine_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% In-memory engine write duration per server", "subtitle": "The time consumed of write in region cache engine per TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_in_memory_engine_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_in_memory_engine_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Prepare for write duration", "subtitle": "The time consumed of prepare for write in the in-memory engine", "expressions": [ "sum(increase( tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% In-memory engine prepare for write duration per server", "subtitle": "The time consumed of prepare for write in the in-memory engine per TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le, $additional_groupby) ))" ] }, { "title": "Iterator operations", "subtitle": "The count of different type of iteration operations", "expressions": [ "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek_found\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next_found\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev\"} [$__rate_interval] )) by ($additional_groupby)", "sum(rate( tikv_in_memory_engine_locate {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev_found\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "Seek duration", "subtitle": "The time consumed when executing seek operation", "expressions": [ "histogram_quantile(1,( sum(rate( tikv_in_memory_engine_seek_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_in_memory_engine_seek_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.95,( sum(rate( tikv_in_memory_engine_seek_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_in_memory_engine_seek_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) / sum(rate( tikv_in_memory_engine_seek_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) )" ] }, { "title": "Oldest Auto GC SafePoint", "subtitle": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one", "expressions": [ "max(( tikv_in_memory_engine_oldest_safe_point {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18)" ] }, { "title": "Newest Auto GC SafePoint", "subtitle": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one", "expressions": [ "max(( tikv_in_memory_engine_newest_safe_point {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18)" ] }, { "title": "Auto GC SafePoint Gap", "subtitle": "The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine", "expressions": [ "(sum(( tikv_in_memory_engine_newest_safe_point {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18) - sum(( tikv_in_memory_engine_oldest_safe_point {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18))" ] }, { "title": "Auto GC SafePoint Gap With TiKV", "subtitle": "The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point", "expressions": [ "max(( tikv_safe_point_gap_with_in_memory_engine {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18)" ] }, { "title": "Cached Region Coprocessor Requests", "subtitle": "Tthe number of coprocessor requests of cached regions that is observed during auto load and evict", "expressions": [ "sum(increase( tikv_in_memory_engine_auto_load_evict_cached_region_coprocessor_requests_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Cached Region MVCC Amplification", "subtitle": "Tthe MVCC amplification of cached regions that is observed during auto load and evict", "expressions": [ "sum(increase( tikv_in_memory_engine_auto_load_evict_cached_region_mvcc_amplification_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Top Region Coprocessor Requests", "subtitle": "Tthe number of coprocessor requests of top regions that is observed during auto load and evict", "expressions": [ "sum(increase( tikv_in_memory_engine_auto_load_evict_top_region_coprocessor_requests_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Top Region MVCC Amplification", "subtitle": "Tthe MVCC amplification of top regions that is observed during auto load and evict", "expressions": [ "sum(increase( tikv_in_memory_engine_auto_load_evict_top_region_mvcc_amplification_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] } ] }, { "title": "Flow Control", "panels": [ { "title": "Scheduler flow", "subtitle": null, "expressions": [ "sum(( tikv_scheduler_write_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_scheduler_throttle_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance) != 0" ] }, { "title": "Scheduler discard ratio", "subtitle": null, "expressions": [ "sum(( tikv_scheduler_discard_ratio {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type) / 10000000" ] }, { "title": "Throttle duration", "subtitle": null, "expressions": [ "sum(increase( tikv_scheduler_throttle_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Scheduler throttled CF", "subtitle": null, "expressions": [ "(( tikv_scheduler_throttle_cf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) != 0" ] }, { "title": "Flow controller actions", "subtitle": null, "expressions": [ "sum(rate( tikv_scheduler_throttle_action_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, cf, $additional_groupby)" ] }, { "title": "Flush/L0 flow", "subtitle": null, "expressions": [ "sum(( tikv_scheduler_l0_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, cf)", "sum(( tikv_scheduler_flush_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, cf)", "sum(( tikv_scheduler_l0_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_scheduler_flush_flow {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Flow controller factors", "subtitle": null, "expressions": [ "max(( tikv_scheduler_l0 {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "max(( tikv_scheduler_memtable {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "max(( tikv_scheduler_l0_avg {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Compaction pending bytes", "subtitle": null, "expressions": [ "sum(( tikv_engine_pending_compaction_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"} )) by (cf, $additional_groupby)" ] }, { "title": "Txn command throttled duration", "subtitle": "Throttle time for txn storage commands in 1 minute.", "expressions": [ "sum(rate( tikv_txn_command_throttle_time_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Non-txn command throttled duration", "subtitle": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", "expressions": [ "sum(rate( tikv_non_txn_command_throttle_time_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] } ] }, { "title": "Scheduler", "panels": [ { "title": "Scheduler stage total", "subtitle": "The total number of commands on each stage", "expressions": [ "sum(rate( tikv_scheduler_too_busy_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (stage, $additional_groupby)", "sum(rate( tikv_scheduler_stage_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (stage, $additional_groupby)" ] }, { "title": "Scheduler priority commands", "subtitle": "The count of different priority commands", "expressions": [ "sum(rate( tikv_scheduler_commands_pri_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (priority, $additional_groupby)" ] }, { "title": "Scheduler pending commands", "subtitle": "The count of pending commands per TiKV instance", "expressions": [ "sum(( tikv_scheduler_contex_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Scheduler running commands", "subtitle": "The count of running commands per TiKV instance", "expressions": [ "sum(( tikv_scheduler_running_commands {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Scheduler writing bytes", "subtitle": "The total writing bytes of commands on each stage", "expressions": [ "sum(( tikv_scheduler_writing_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Scheduler memory quota", "subtitle": "The number of bytes used by scheduler", "expressions": [ "sum(( tikv_scheduler_memory_quota_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"} )) by (instance)" ] }, { "title": "Txn Scheduler Pool Wait Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_yatp_pool_schedule_wait_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le)" ] }, { "title": "Txn Scheduler Pool Exec Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_yatp_pool_schedule_exec_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le)" ] } ] }, { "title": "Scheduler Worker Pool", "panels": [ { "title": "Time used by level", "subtitle": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", "expressions": [ "sum(rate( tikv_multilevel_level_elapsed {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (level, $additional_groupby)" ] }, { "title": "Level 0 chance", "subtitle": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", "expressions": [ "(( tikv_multilevel_level0_chance {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} ))" ] }, { "title": "Running tasks", "subtitle": "The number of concurrently running tasks in the yatp thread pool.", "expressions": [ "sum(avg_over_time( tikv_scheduler_running_commands {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (instance, $additional_groupby)" ] }, { "title": "Wait Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_yatp_pool_schedule_wait_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le)" ] }, { "title": "Running threads", "subtitle": "The number of concurrently running threads in the yatp thread pool.", "expressions": [ "sum(avg_over_time( tikv_unified_read_pool_thread_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (instance, $additional_groupby)" ] }, { "title": "Duration of One Time Slice", "subtitle": "Task execution time during one schedule.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_poll_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_poll_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_poll_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_poll_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Task Execute Duration", "subtitle": "Task total execution duration.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_exec_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_exec_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_exec_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_exec_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Task Schedule Times", "subtitle": "Task schedule number of times.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_execute_times_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_execute_times_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_execute_times_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_execute_times_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] } ] }, { "title": "Scheduler - $command", "panels": [ { "title": "Scheduler stage total", "subtitle": "The total number of commands on each stage in commit command", "expressions": [ "sum(rate( tikv_scheduler_too_busy_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_scheduler_stage_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (stage, $additional_groupby)" ] }, { "title": "Scheduler command duration", "subtitle": "The time consumed when executing commit command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_command_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_command_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_command_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_command_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Scheduler latch wait duration", "subtitle": "The time which is caused by latch wait in commit command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_latch_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_latch_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_latch_wait_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_latch_wait_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Scheduler keys read", "subtitle": "The count of keys read by a commit command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_kv_command_key_read_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_kv_command_key_read_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_kv_command_key_read_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_kv_command_key_read_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Scheduler keys written", "subtitle": "The count of keys written by a commit command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_kv_command_key_write_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_kv_command_key_write_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_kv_command_key_write_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_kv_command_key_write_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Scheduler scan details", "subtitle": "The keys scan details of each CF when executing commit command", "expressions": [ "sum(rate( tikv_scheduler_kv_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Scheduler scan details [lock]", "subtitle": "The keys scan details of lock CF when executing commit command", "expressions": [ "sum(rate( tikv_scheduler_kv_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Scheduler scan details [write]", "subtitle": "The keys scan details of write CF when executing commit command", "expressions": [ "sum(rate( tikv_scheduler_kv_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Scheduler scan details [default]", "subtitle": "The keys scan details of default CF when executing commit command", "expressions": [ "sum(rate( tikv_scheduler_kv_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Scheduler command process duration", "subtitle": "The time consumed on processing command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_processing_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_processing_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_processing_read_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_processing_read_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Scheduler command block read duration", "subtitle": "The time consumed on rocksdb block read while processing command", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_scheduler_block_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_scheduler_block_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_scheduler_block_read_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_scheduler_block_read_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Check memory locks duration", "subtitle": "The time consumed on checking memory locks", "expressions": [ "sum(increase( tikv_storage_check_mem_lock_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le)" ] } ] }, { "title": "Coprocessor Overview", "panels": [ { "title": "Request duration", "subtitle": "The time consumed to handle coprocessor read requests", "expressions": [ "sum(increase( tikv_coprocessor_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Request duration", "subtitle": "The time consumed to handle coprocessor read requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "(sum(rate( tikv_coprocessor_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby) / sum(rate( tikv_coprocessor_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby) )" ] }, { "title": "Total Requests", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby)" ] }, { "title": "Total Request Errors", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_request_error {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (reason, $additional_groupby)" ] }, { "title": "KV Cursor Operations", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_scan_keys_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby)" ] }, { "title": "KV Cursor Operations", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_scan_keys_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_scan_keys_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "(sum(rate( tikv_coprocessor_scan_keys_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby) / sum(rate( tikv_coprocessor_scan_keys_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, $additional_groupby) )" ] }, { "title": "Total RocksDB Perf Statistics", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"} [$__rate_interval] )) by (req, $additional_groupby)" ] }, { "title": "Total Response Size", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_response_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Memory Quota", "subtitle": "Total bytes of memory used by coprocessor requests", "expressions": [ "sum(( tikv_coprocessor_memory_quota {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, type)" ] } ] }, { "title": "Coprocessor Detail", "panels": [ { "title": "Handle duration", "subtitle": "The time consumed when handling coprocessor requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))" ] }, { "title": "Handle duration by store", "subtitle": "The time consumed to handle coprocessor requests per TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, instance, le) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_handle_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (req, instance, le) ))" ] }, { "title": "Wait duration", "subtitle": "The time consumed when coprocessor requests are wait for being handled", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))" ] }, { "title": "Wait duration by store", "subtitle": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"} [$__rate_interval] )) by (req, instance, le) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_wait_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"} [$__rate_interval] )) by (req, instance, le) ))" ] }, { "title": "Total DAG Requests", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_dag_request_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (vec_type, $additional_groupby)" ] }, { "title": "Total DAG Executors", "subtitle": "The total number of DAG executors", "expressions": [ "sum(rate( tikv_coprocessor_executor_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Total Ops Details (Table Scan)", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Total Ops Details (Index Scan)", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"} [$__rate_interval] )) by (tag, $additional_groupby)" ] }, { "title": "Total Ops Details by CF (Table Scan)", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"} [$__rate_interval] )) by (cf, tag, $additional_groupby)" ] }, { "title": "Total Ops Details by CF (Index Scan)", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_scan_details {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"} [$__rate_interval] )) by (cf, tag, $additional_groupby)" ] }, { "title": "Memory lock checking duration", "subtitle": "The time consumed on checking memory locks for coprocessor requests", "expressions": [ "sum(increase( tikv_coprocessor_mem_lock_check_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Memory lock checking duration", "subtitle": "The time consumed on checking memory locks for coprocessor requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_mem_lock_check_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_mem_lock_check_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_coprocessor_mem_lock_check_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_coprocessor_mem_lock_check_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Semaphore waiting duration", "subtitle": "The time consumed on waiting for semaphore permits for heavy coprocessor requests", "expressions": [ "sum(increase( tikv_coprocessor_semaphore_wait_time_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Semaphore waiting duration", "subtitle": "The time consumed on waiting for semaphore permits for heavy coprocessor requests", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_semaphore_wait_time_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_semaphore_wait_time_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_coprocessor_semaphore_wait_time_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_coprocessor_semaphore_wait_time_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Semaphore waiting tasks count", "subtitle": "The number of cop tasks waiting for semaphore permits.", "expressions": [ "sum(avg_over_time( tikv_coprocessor_waiting_for_semaphore {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30s] )) by (instance, $additional_groupby)" ] } ] }, { "title": "Unified Read Pool", "panels": [ { "title": "Time used by level", "subtitle": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", "expressions": [ "sum(rate( tikv_multilevel_level_elapsed {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (level, $additional_groupby)" ] }, { "title": "Level 0 chance", "subtitle": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", "expressions": [ "(( tikv_multilevel_level0_chance {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} ))" ] }, { "title": "Running tasks", "subtitle": "The number of concurrently running tasks in the yatp thread pool.", "expressions": [ "sum(avg_over_time( tikv_unified_read_pool_running_tasks {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (priority, $additional_groupby)" ] }, { "title": "Wait Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_yatp_pool_schedule_wait_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le)" ] }, { "title": "Running threads", "subtitle": "The number of concurrently running threads in the yatp thread pool.", "expressions": [ "sum(avg_over_time( tikv_unified_read_pool_thread_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (instance, $additional_groupby)" ] }, { "title": "Duration of One Time Slice", "subtitle": "Task execution time during one schedule.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_poll_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_poll_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_poll_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_poll_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Task Execute Duration", "subtitle": "Task total execution duration.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_exec_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_exec_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_exec_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_exec_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Task Schedule Times", "subtitle": "Task schedule number of times.", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_yatp_task_execute_times_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_yatp_task_execute_times_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_yatp_task_execute_times_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_yatp_task_execute_times_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"} [$__rate_interval] )) by ($additional_groupby) )" ] } ] }, { "title": "GC", "panels": [ { "title": "GC tasks", "subtitle": "The count of GC tasks processed by gc_worker", "expressions": [ "sum(rate( tikv_gcworker_gc_tasks_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (task, $additional_groupby)", "sum(rate( tikv_storage_gc_skipped_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (task, $additional_groupby)", "sum(rate( tikv_gcworker_gc_task_fail_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (task, $additional_groupby)", "sum(rate( tikv_gc_worker_too_busy {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "GC tasks duration", "subtitle": "The time consumed when executing GC tasks", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_gcworker_gc_task_duration_vec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_gcworker_gc_task_duration_vec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_gcworker_gc_task_duration_vec_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_gcworker_gc_task_duration_vec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "TiDB GC seconds", "subtitle": "The GC duration", "expressions": [ "histogram_quantile(1,( sum(rate( tidb_tikvclient_gc_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "TiDB GC worker actions", "subtitle": "The count of TiDB GC worker actions", "expressions": [ "sum(rate( tidb_tikvclient_gc_worker_actions_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "ResolveLocks Progress", "subtitle": "Progress of ResolveLocks, the first phase of GC", "expressions": [ "max(( tidb_tikvclient_range_task_stats {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=~\"resolve-locks.*\"} )) by (result, $additional_groupby)" ] }, { "title": "TiKV Auto GC Progress", "subtitle": "Progress of TiKV's GC", "expressions": [ "(sum(( tikv_gcworker_autogc_processed_regions {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"} )) by (instance) / sum(( tikv_raftstore_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"} )) by (instance) )" ] }, { "title": "GC speed", "subtitle": "keys / second", "expressions": [ "sum(rate( tikv_storage_mvcc_gc_delete_versions_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)" ] }, { "title": "TiKV Auto GC SafePoint", "subtitle": "SafePoint used for TiKV's Auto GC", "expressions": [ "max(( tikv_gcworker_autogc_safe_point {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"} )) by (instance, $additional_groupby) / (2^18)" ] }, { "title": "GC lifetime", "subtitle": "The lifetime of TiDB GC", "expressions": [ "max(( tidb_tikvclient_gc_config {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_life_time\"} ))" ] }, { "title": "GC interval", "subtitle": "The interval of TiDB GC", "expressions": [ "max(( tidb_tikvclient_gc_config {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"} ))" ] }, { "title": "GC in Compaction Filter", "subtitle": "Keys handled in GC compaction filter", "expressions": [ "sum(rate( tikv_gc_compaction_filtered {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_skip {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_mvcc_rollback {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_orphan_versions {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_perform {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_failure {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, type, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_mvcc_deletion_met {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_mvcc_deletion_handled {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)", "sum(rate( tikv_gc_compaction_filter_mvcc_deletion_wasted {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (key_mode, $additional_groupby)" ] }, { "title": "GC scan write details", "subtitle": "GC scan write details", "expressions": [ "sum(rate( tikv_gcworker_gc_keys {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"} [$__rate_interval] )) by (key_mode, tag, $additional_groupby)" ] }, { "title": "GC scan default details", "subtitle": "GC scan default details", "expressions": [ "sum(rate( tikv_gcworker_gc_keys {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"} [$__rate_interval] )) by (key_mode, tag, $additional_groupby)" ] }, { "title": "Auto Compaction Duration", "subtitle": "Time spent on auto compaction operations by type", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_auto_compaction_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_auto_compaction_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "(sum(rate( tikv_auto_compaction_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) / sum(rate( tikv_auto_compaction_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) )" ] }, { "title": "Auto Compaction Regions Status", "subtitle": "Number of regions meeting compaction threshold and pending candidates", "expressions": [ "sum(( tikv_auto_compaction_regions_meet_threshold {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_auto_compaction_pending_candidates {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] } ] }, { "title": "Pessimistic Locking", "panels": [ { "title": "Lock Manager Thread CPU", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"} [$__rate_interval] )) by (instance, name)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"} [$__rate_interval] )) by (instance, name)" ] }, { "title": "Lock Manager Handled tasks", "subtitle": null, "expressions": [ "sum(rate( tikv_lock_manager_task_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Waiter lifetime duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_lock_manager_waiter_lifetime_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_lock_manager_waiter_lifetime_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_lock_manager_waiter_lifetime_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_lock_manager_waiter_lifetime_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Lock Waiting Queue", "subtitle": null, "expressions": [ "sum(max_over_time( tikv_lock_manager_wait_table_status {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30s] )) by (type, $additional_groupby)", "sum(max_over_time( tikv_lock_wait_queue_entries_gauge_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30s] )) by (type, $additional_groupby)" ] }, { "title": "Deadlock detect duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_lock_manager_detect_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_lock_manager_detect_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_lock_manager_detect_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_lock_manager_detect_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] }, { "title": "Detect error", "subtitle": null, "expressions": [ "sum(rate( tikv_lock_manager_error_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "Deadlock detector leader", "subtitle": null, "expressions": [ "sum(max_over_time( tikv_lock_manager_detector_leader_heartbeat {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30s] )) by (instance)" ] }, { "title": "Total pessimistic locks memory size", "subtitle": null, "expressions": [ "(( tikv_pessimistic_lock_memory_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "In-memory pessimistic locking result", "subtitle": null, "expressions": [ "sum(rate( tikv_in_memory_pessimistic_locking {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (result, $additional_groupby)" ] }, { "title": "Pessimistic lock activities", "subtitle": "The number of active keys and waiters.", "expressions": [ "sum(( tikv_lock_wait_queue_entries_gauge_vec {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Lengths of lock wait queues when transaction enqueues", "subtitle": "The length includes the entering transaction itself", "expressions": [ "sum(increase( tikv_lock_wait_queue_length_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "In-memory scan lock read duration", "subtitle": "The duration scan in-memory pessimistic locks with read lock", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))" ] } ] }, { "title": "Task", "panels": [ { "title": "Worker handled tasks", "subtitle": "The number of tasks handled by worker", "expressions": [ "sum(rate( tikv_worker_handled_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, $additional_groupby)" ] }, { "title": "Worker pending tasks", "subtitle": "Current pending and running tasks of worker", "expressions": [ "sum(( tikv_worker_pending_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (name, $additional_groupby)" ] }, { "title": "FuturePool handled tasks", "subtitle": "The number of tasks handled by future_pool", "expressions": [ "sum(rate( tikv_futurepool_handled_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, $additional_groupby)" ] }, { "title": "FuturePool pending tasks", "subtitle": "Current pending and running tasks of future_pool", "expressions": [ "sum(avg_over_time( tikv_futurepool_pending_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (name, $additional_groupby)" ] } ] }, { "title": "PD", "panels": [ { "title": "PD requests", "subtitle": "The count of requests that TiKV sends to PD", "expressions": [ "sum(rate( tikv_pd_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "PD request duration (average)", "subtitle": "The time consumed by requests that TiKV sends to PD", "expressions": [ "(sum(rate( tikv_pd_request_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) / sum(rate( tikv_pd_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby) )" ] }, { "title": "PD heartbeats", "subtitle": "The total number of PD heartbeat messages", "expressions": [ "sum(rate( tikv_pd_heartbeat_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)", "sum(( tikv_pd_pending_heartbeat_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "PD validate peers", "subtitle": "The total number of peers validated by the PD worker", "expressions": [ "sum(rate( tikv_pd_validate_peer_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "PD reconnection", "subtitle": "The count of reconnection between TiKV and PD", "expressions": [ "sum(delta( tikv_pd_reconnect_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "PD forward status", "subtitle": "The forward status of PD client", "expressions": [ "(( tikv_pd_request_forwarded {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Pending TSO Requests", "subtitle": "The number of TSO requests waiting in the queue.", "expressions": [ "sum(( tikv_pd_pending_tso_request_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, $additional_groupby)" ] } ] }, { "title": "Slow Trend Statistics", "panels": [ { "title": "Inspected duration per server", "subtitle": "The duration that recorded by inspecting messages.", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_inspect_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type, le) ))" ] }, { "title": "Store Slow Score", "subtitle": "The slow score of stores", "expressions": [ "sum(( tikv_raftstore_slow_score {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, type)" ] }, { "title": "Slow Trend", "subtitle": "The changing trend of the slowness on I/O operations. 'value > 0' means the related store might have a slow trend.", "expressions": [ "sum(( tikv_raftstore_slow_trend {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "QPS Changing Trend", "subtitle": "The changing trend of QPS on each store. 'value < 0' means the QPS has a dropping trend.", "expressions": [ "sum(( tikv_raftstore_slow_trend_result {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "AVG Sampling Latency", "subtitle": "The sampling latency of recent queries. A larger value indicates that the store is more likely to be the slowest store.", "expressions": [ "sum(( tikv_raftstore_slow_trend_l0 {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "QPS of each store", "subtitle": "The QPS of each store.", "expressions": [ "sum(( tikv_raftstore_slow_trend_result_value {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Inspected network duration per server", "subtitle": "The duration that recorded by inspecting network. Note: it will be collected only when duration is greater than 100ms, to avoid collect too many metrics.", "expressions": [ "histogram_quantile(0.8,( sum(rate( tikv_raftstore_inspect_network_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (source, type, le) ))" ] } ] }, { "title": "Snapshot", "panels": [ { "title": "Rate snapshot message", "subtitle": "The rate of Raft snapshot messages sent", "expressions": [ "sum(delta( tikv_raftstore_raft_sent_message_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [1m] )) by (instance)" ] }, { "title": "Snapshot state count", "subtitle": "The number of snapshots in different states", "expressions": [ "sum(( tikv_raftstore_snapshot_traffic_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "99% Snapshot generation/apply wait duration", "subtitle": "The time snapshot generation/apply tasks spent waiting to be executed.", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "99% Handle snapshot duration", "subtitle": "The time consumed when handling snapshots", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_server_send_snapshot_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_snapshot_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_raftstore_snapshot_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "99.99% Snapshot size", "subtitle": "The snapshot size (P99.99).9999", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_snapshot_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "99.99% Snapshot KV count", "subtitle": "The number of KV within a snapshot in .9999", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_snapshot_kv_count_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Snapshot Actions", "subtitle": "Action stats for snapshot generating and applying", "expressions": [ "sum(delta( tikv_raftstore_snapshot_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, status, $additional_groupby)", "sum(delta( tikv_raftstore_clean_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, status, $additional_groupby)", "sum(delta( tikv_server_snapshot_task_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, $additional_groupby)" ] }, { "title": "Snapshot transport speed", "subtitle": "The speed of sending or receiving snapshot", "expressions": [ "sum(rate( tikv_snapshot_limit_transport_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)", "sum(rate( tikv_snapshot_limit_generate_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Snapshot pending applies", "subtitle": "The number of snapshots waiting to be applied", "expressions": [ "(( tikv_raftstore_snapshot_pending_applies {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] } ] }, { "title": "Resolved TS", "panels": [ { "title": "Resolved TS Worker CPU", "subtitle": "The CPU utilization of resolved ts worker", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Advance ts Worker CPU", "subtitle": "The CPU utilization of advance ts worker", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Scan lock Worker CPU", "subtitle": "The CPU utilization of scan lock worker", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Max gap of resolved-ts", "subtitle": "The gap between resolved ts (the maximum candidate of safe-ts) and current time.", "expressions": [ "sum(( tikv_resolved_ts_min_resolved_ts_gap_millis {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Min Resolved TS Region", "subtitle": "The region that has minimal resolved ts", "expressions": [ "sum(( tikv_resolved_ts_min_resolved_ts_region {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Max gap of follower safe-ts", "subtitle": "The gap between now() and the minimal (non-zero) safe ts for followers", "expressions": [ "sum(( tikv_resolved_ts_min_follower_safe_ts_gap_millis {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Min Safe TS Follower Region", "subtitle": "The region id of the follower that has minimal safe ts", "expressions": [ "sum(( tikv_resolved_ts_min_follower_safe_ts_region {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Max gap of resolved-ts in region leaders", "subtitle": "The gap between resolved ts of leaders and current time", "expressions": [ "sum(( tikv_resolved_ts_min_leader_resolved_ts_gap_millis {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Min Leader Resolved TS Region", "subtitle": "The region that its leader has minimal resolved ts.", "expressions": [ "sum(( tikv_resolved_ts_min_leader_resolved_ts_region {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Check leader duration", "subtitle": "The time consumed when handle a check leader request", "expressions": [ "sum(increase( tikv_resolved_ts_check_leader_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "99% CheckLeader request region count", "subtitle": "Bucketed histogram of region count in a check leader request", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_check_leader_request_item_count_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "99% CheckLeader request size", "subtitle": "Bucketed histogram of the check leader request size", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_check_leader_request_size_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))", "histogram_quantile(0.99,( sum(rate( tikv_check_leader_request_item_count_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, le) ))" ] }, { "title": "Fail advance ts count", "subtitle": "The count of fail to advance resolved-ts", "expressions": [ "sum(delta( tikv_resolved_ts_fail_advance_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, reason)", "sum(delta( tikv_raftstore_check_stale_peer {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Lock heap size", "subtitle": "Total bytes in memory of resolved-ts observe regions's lock heap", "expressions": [ "avg(( tikv_resolved_ts_lock_heap_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Initial scan backoff duration", "subtitle": "The backoff duration before starting initial scan", "expressions": [ "sum(increase( tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Observe region status", "subtitle": "The status of resolved-ts observe regions", "expressions": [ "sum(( tikv_resolved_ts_region_resolve_status {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby)" ] }, { "title": "Pending command size", "subtitle": "Total bytes of pending commands in the channel", "expressions": [ "avg(( tikv_resolved_ts_channel_penging_cmd_bytes_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] } ] }, { "title": "Point In Time Restore", "panels": [ { "title": "CPU Usage", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "P99 RPC Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_import_apply_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_import_engine_request_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))" ] }, { "title": "Import RPC Ops", "subtitle": null, "expressions": [ "sum(rate( tikv_import_rpc_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"} [$__rate_interval] )) by (instance, request)", "sum(rate( tikv_import_rpc_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"} [$__rate_interval] )) by (request)" ] }, { "title": "Import RPC Count", "subtitle": null, "expressions": [ "(( tikv_import_rpc_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} ))" ] }, { "title": "Cache Events", "subtitle": null, "expressions": [ "sum(rate( tikv_import_apply_cache_event {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, instance)" ] }, { "title": "Overall RPC Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Read File into Memory Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_apply_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"} [$__rate_interval] )) by (le)" ] }, { "title": "Queuing Time", "subtitle": null, "expressions": [ "sum(increase( tikv_import_engine_request_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply Request Throughput", "subtitle": null, "expressions": [ "sum(rate( tikv_import_apply_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Downloaded File Size", "subtitle": null, "expressions": [ "sum(increase( tikv_import_download_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply Batch Size", "subtitle": null, "expressions": [ "sum(increase( tikv_import_apply_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Blocked by Concurrency Time", "subtitle": null, "expressions": [ "sum(increase( tikv_import_engine_request_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"} [$__rate_interval] )) by (le)" ] }, { "title": "Apply Request Speed", "subtitle": null, "expressions": [ "sum(rate( tikv_import_applier_event {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Cached File in Memory", "subtitle": null, "expressions": [ "sum(( tikv_import_apply_cached_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Engine Requests Unfinished", "subtitle": null, "expressions": [ "sum(rate( tikv_import_applier_event {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"} [$__rate_interval] )) by (instance, type)" ] }, { "title": "Apply Time", "subtitle": null, "expressions": [ "sum(increase( tikv_import_engine_request_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"} [$__rate_interval] )) by (le)" ] }, { "title": "Raft Store Memory Usage", "subtitle": null, "expressions": [ "sum(( tikv_server_mem_trace_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"} )) by (instance)" ] } ] }, { "title": "Backup & Import", "panels": [ { "title": "Backup CPU Utilization", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"} [$__rate_interval] )) by (instance)", "(( tikv_backup_softlimit {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Backup Thread Count", "subtitle": null, "expressions": [ "sum(( tikv_backup_thread_pool_size {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Backup Errors", "subtitle": null, "expressions": [ "sum(delta( tikv_backup_error_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, error)" ] }, { "title": "Backup Write CF SST Size", "subtitle": null, "expressions": [ "sum(increase( tikv_backup_range_size_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"} [$__rate_interval] )) by (le)" ] }, { "title": "Backup Default CF SST Size", "subtitle": null, "expressions": [ "sum(increase( tikv_backup_range_size_bytes_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"} [$__rate_interval] )) by (le)" ] }, { "title": "Backup SST Generation Throughput", "subtitle": null, "expressions": [ "sum(rate( tikv_backup_range_size_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] ))", "sum(rate( tikv_backup_range_size_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, cf)" ] }, { "title": "Backup Scan SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_backup_range_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"} [$__rate_interval] )) by (le)" ] }, { "title": "Backup Scan SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_backup_range_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"} [$__rate_interval] )) by (le)" ] }, { "title": "Backup Save SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_backup_range_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"} [$__rate_interval] )) by (le)" ] }, { "title": "Backup SST Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.999,( sum(rate( tikv_backup_range_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_backup_range_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, le, $additional_groupby) ))", "(sum(( tikv_backup_range_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby) / sum(( tikv_backup_range_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (type, $additional_groupby) )" ] }, { "title": "External Storage Create Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_external_storage_create_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "External Storage Create Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_external_storage_create_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_external_storage_create_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))" ] }, { "title": "Checksum Request Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_coprocessor_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_coprocessor_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"} [$__rate_interval] )) by (req, le, $additional_groupby) ))" ] }, { "title": "IO Utilization", "subtitle": null, "expressions": [ "sum(rate( node_disk_io_time_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, device)" ] }, { "title": "Import CPU Utilization", "subtitle": null, "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Import Thread Count", "subtitle": null, "expressions": [ "count(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Import Errors", "subtitle": null, "expressions": [ "sum(delta( tikv_import_error_counter {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, error, instance)" ] }, { "title": "Import RPC Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (request, le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (request, le, $additional_groupby) ))", "(sum(rate( tikv_import_rpc_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (request, $additional_groupby) / sum(rate( tikv_import_rpc_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (request, $additional_groupby) )" ] }, { "title": "Import RPC Ops", "subtitle": null, "expressions": [ "sum(rate( tikv_import_rpc_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"} [$__rate_interval] )) by (request, $additional_groupby)" ] }, { "title": "Import RPC Count", "subtitle": null, "expressions": [ "(( tikv_import_rpc_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Import Write/Download RPC Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Wait Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_download_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Read SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_download_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Rewrite SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_download_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Ingest RPC Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_rpc_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Ingest SST Duration", "subtitle": null, "expressions": [ "sum(increase( tikv_import_ingest_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Ingest SST Bytes", "subtitle": null, "expressions": [ "sum(increase( tikv_import_ingest_byte_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Import Download SST Throughput", "subtitle": null, "expressions": [ "sum(rate( tikv_import_download_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)", "sum(rate( tikv_import_download_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] ))" ] }, { "title": "Import Local Write keys", "subtitle": null, "expressions": [ "sum(delta( tikv_import_local_write_keys {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, instance)" ] }, { "title": "Import Local Write bytes", "subtitle": null, "expressions": [ "sum(rate( tikv_import_local_write_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, instance)" ] }, { "title": "TTL Expired", "subtitle": null, "expressions": [ "sum(( tikv_backup_raw_expired_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_backup_raw_expired_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "cloud request", "subtitle": null, "expressions": [ "sum(rate( tikv_cloud_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (cloud, req)" ] } ] }, { "title": "Backup Log", "panels": [ { "title": "Endpoint Status", "subtitle": null, "expressions": [ "(( tikv_log_backup_enabled {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Task Status", "subtitle": null, "expressions": [ "min(( tikv_log_backup_task_status {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Advancer Owner", "subtitle": null, "expressions": [ "tidb_log_backup_advancer_owner > 0" ] }, { "title": "Average Flush Size", "subtitle": "The average flush size of last 30mins.", "expressions": [ "(sum(increase( tikv_log_backup_flush_file_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30m] )) by (instance) / sum(increase( tikv_log_backup_flush_duration_sec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"} [30m] )) by (instance) )" ] }, { "title": "Flushed Files (Last 30m) Per Host", "subtitle": "The current total flushed file number of this run.", "expressions": [ "sum(delta( tikv_log_backup_flush_file_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30m] )) by (instance) > 0" ] }, { "title": "Flush Times (Last 30m)", "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "sum(delta( tikv_log_backup_flush_duration_sec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"} [30m] )) by (instance)" ] }, { "title": "Total Flushed Size (Last 30m)", "subtitle": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "sum(delta( tikv_log_backup_flush_file_size_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30m] )) by (instance)" ] }, { "title": "Flush Files (Last 30m)", "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "sum(delta( tikv_log_backup_flush_file_size_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [30m] )) by (instance)" ] }, { "title": "CPU Usage", "subtitle": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", "expressions": [ "sum(rate( tikv_thread_cpu_seconds_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Handle Event Rate", "subtitle": null, "expressions": [ "sum(rate( tikv_log_backup_handle_kv_batch_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Initial Scan Generate Event Throughput", "subtitle": "The data rate of initial scanning emitting events.", "expressions": [ "sum(rate( tikv_log_backup_incremental_scan_bytes_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "Abnormal Checkpoint TS Lag", "subtitle": null, "expressions": [ "(time() * 1000 - max(( tidb_log_backup_last_checkpoint {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (task) / 262144 > 0)" ] }, { "title": "Memory Of Events", "subtitle": "The estimated memory usage by the streaming backup module.", "expressions": [ "sum(( tikv_log_backup_heap_memory {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Observed Region Count", "subtitle": null, "expressions": [ "sum(( tikv_log_backup_observed_region {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)", "sum(( tikv_log_backup_observed_region {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Errors", "subtitle": "The errors met when backing up.\n**They are retryable, don't worry.**", "expressions": [ "sum(delta( tikv_log_backup_errors {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, instance)" ] }, { "title": "Fatal Errors", "subtitle": "The errors met when backing up.", "expressions": [ "sum(delta( tikv_log_backup_fatal_errors {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [1m] )) by (type, instance)" ] }, { "title": "Checkpoint TS of Tasks", "subtitle": null, "expressions": [ "max(( tidb_log_backup_last_checkpoint {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (task) / 262144 > 0", "time() * 1000" ] }, { "title": "Flush Duration", "subtitle": "The duration of flushing a batch of file.", "expressions": [ "sum(increase( tikv_log_backup_flush_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"} [$__rate_interval] )) by (le)" ] }, { "title": "Initial scanning duration", "subtitle": "The duration of scanning the initial data from local DB and transform them into apply events.", "expressions": [ "sum(increase( tikv_log_backup_initial_scan_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Convert Raft Event duration", "subtitle": "The duration of converting a raft request into a apply event.", "expressions": [ "sum(increase( tikv_log_backup_event_handle_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"} [$__rate_interval] )) by (le)" ] }, { "title": "Wait for Lock Duration", "subtitle": "The duration of waiting the mutex of the controller.", "expressions": [ "sum(increase( tikv_log_backup_event_handle_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"} [$__rate_interval] )) by (le)" ] }, { "title": "Command Batch Size", "subtitle": "The number of KV-modify of each raft command observed.", "expressions": [ "sum(increase( tikv_log_backup_handle_kv_batch_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le)" ] }, { "title": "Save to Temp File Duration", "subtitle": "The total cost of saving an event into temporary file.", "expressions": [ "sum(increase( tikv_log_backup_event_handle_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"} [$__rate_interval] )) by (le)" ] }, { "title": "Write to Temp File Duration", "subtitle": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.", "expressions": [ "sum(increase( tikv_log_backup_on_event_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"} [$__rate_interval] )) by (le)" ] }, { "title": "System Write Call Duration", "subtitle": "The duration of collecting metadata and call the UNIX system call *write* for each event.", "expressions": [ "sum(increase( tikv_log_backup_on_event_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"} [$__rate_interval] )) by (le)" ] }, { "title": "Internal Message Type", "subtitle": "The internal message type count.", "expressions": [ "sum(rate( tikv_log_backup_internal_actor_acting_duration_sec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (message)" ] }, { "title": "Internal Message Handling Duration (P99)", "subtitle": "The internal handling message duration.", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_log_backup_internal_actor_acting_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (message, le) ))" ] }, { "title": "Internal Message Handling Duration (P90)", "subtitle": "The internal handling message duration.", "expressions": [ "histogram_quantile(0.9,( sum(rate( tikv_log_backup_internal_actor_acting_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (message, le) ))" ] }, { "title": "Initial Scan RocksDB Throughput", "subtitle": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", "expressions": [ "sum(rate( tikv_log_backup_initial_scan_operations {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"} [$__rate_interval] )) by (cf)" ] }, { "title": "Initial Scan RocksDB Operation", "subtitle": "Misc statistics of RocksDB during initial scanning.", "expressions": [ "sum(rate( tikv_log_backup_initial_scan_operations {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"} [$__rate_interval] )) by (cf, op) > 0" ] }, { "title": "Initial Scanning Trigger Reason", "subtitle": "The reason of triggering initial scanning.", "expressions": [ "sum(rate( tikv_log_backup_initial_scan_reason {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (reason)" ] }, { "title": "Initial Scanning Task Status", "subtitle": "The task status of initial scanning.", "expressions": [ "sum(( tikv_log_backup_pending_initial_scan {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (stage)" ] }, { "title": "Region Checkpoint Key Putting", "subtitle": null, "expressions": [ "sum(rate( tikv_log_backup_metadata_key_operation {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type)" ] }, { "title": "Request Checkpoint Batch Size", "subtitle": null, "expressions": [ "sum(increase( tidb_log_backup_advancer_batch_size_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"} [$__rate_interval] )) by (le)" ] }, { "title": "Tick Duration", "subtitle": null, "expressions": [ "sum(increase( tidb_log_backup_advancer_tick_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"} [$__rate_interval] )) by (le)" ] }, { "title": "Region Checkpoint Failure Reason", "subtitle": "The reason of advancer failed to be advanced.", "expressions": [ "sum(rate( tidb_log_backup_region_request_failure {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"} [$__rate_interval] )) by (reason)" ] }, { "title": "Request Result", "subtitle": "The result of getting region checkpoints.", "expressions": [ "sum(rate( tidb_log_backup_region_request {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (result)" ] }, { "title": "Tick Duration (P99)", "subtitle": "The internal handling message duration.", "expressions": [ "histogram_quantile(0.99,( sum(rate( tidb_log_backup_advancer_tick_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (step, le) ))" ] }, { "title": "Tick Duration (P90)", "subtitle": "The internal handling message duration.", "expressions": [ "histogram_quantile(0.9,( sum(rate( tidb_log_backup_advancer_tick_duration_sec_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (step, le) ))" ] }, { "title": "Get Region Operation Count", "subtitle": "The frequent of getting region level checkpoint.", "expressions": [ "sum(rate( tidb_log_backup_advancer_tick_duration_sec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"} [$__rate_interval] )) by (step, instance)" ] }, { "title": "Try Advance Trigger Time", "subtitle": "The variant of checkpoint group.", "expressions": [ "sum(rate( tidb_log_backup_advancer_tick_duration_sec_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"} [$__rate_interval] )) by (step, instance)" ] } ] }, { "title": "Threads", "panels": [ { "title": "Threads state", "subtitle": null, "expressions": [ "sum(( tikv_threads_state {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, state)", "sum(( tikv_threads_state {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Threads IO", "subtitle": null, "expressions": [ "topk(20,( sum(rate( tikv_threads_io_bytes_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name, io) > 1024 )) by ($additional_groupby)" ] }, { "title": "Thread Voluntary Context Switches", "subtitle": null, "expressions": [ "topk(20,( max(rate( tikv_thread_voluntary_context_switches {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name) > 100 ))" ] }, { "title": "Thread Nonvoluntary Context Switches", "subtitle": null, "expressions": [ "topk(20,( max(rate( tikv_thread_nonvoluntary_context_switches {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (name) > 100 ))" ] } ] }, { "title": "Memory", "panels": [ { "title": "Allocator Stats", "subtitle": null, "expressions": [ "sum(( tikv_allocator_stats {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, type)" ] }, { "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "subtitle": null, "expressions": [ "(sum(rate( tikv_allocator_thread_allocation {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"} [$__rate_interval] )) by (thread_name, $additional_groupby) - sum(rate( tikv_allocator_thread_allocation {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"} [$__rate_interval] )) by (thread_name, $additional_groupby) )" ] }, { "title": "Allocated Bytes Rate per Thread", "subtitle": null, "expressions": [ "sum(rate( tikv_allocator_thread_allocation {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"} [$__rate_interval] )) by (thread_name, $additional_groupby)" ] }, { "title": "Released Bytes Rate per Thread", "subtitle": null, "expressions": [ "sum(rate( tikv_allocator_thread_allocation {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"} [$__rate_interval] )) by (thread_name, $additional_groupby)" ] }, { "title": "Mapped Allocation per Thread", "subtitle": null, "expressions": [ "sum(( tikv_allocator_thread_stats {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"mapped\"} )) by (thread_name, $additional_groupby)" ] }, { "title": "Arena Count", "subtitle": null, "expressions": [ "sum(( tikv_allocator_arena_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] } ] }, { "title": "Resource Control", "panels": [ { "title": "Background Task Total Wait Duration", "subtitle": null, "expressions": [ "sum(rate( tikv_resource_control_background_task_wait_duration {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance, resource_group)" ] }, { "title": "Priority Quota Limit", "subtitle": "The memory usage of the resource control module.", "expressions": [ "sum(( tikv_resource_control_priority_quota_limit {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance, priority)" ] } ] }, { "title": "Status Server", "panels": [ { "title": "Status API Request Duration", "subtitle": "The 99 quantile durtion of status server API requests", "expressions": [ "histogram_quantile(0.99,( sum(rate( tikv_status_server_request_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (path, le, $additional_groupby) ))" ] }, { "title": "Status API Request (op/s)", "subtitle": null, "expressions": [ "sum(rate( tikv_status_server_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (path, $additional_groupby)" ] } ] }, { "title": "Encryption", "panels": [ { "title": "Encryption data keys", "subtitle": "Total number of encryption data keys in use", "expressions": [ "sum(( tikv_encryption_data_key_storage_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Encrypted files", "subtitle": "Number of files being encrypted", "expressions": [ "sum(( tikv_encryption_file_num {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} )) by (instance)" ] }, { "title": "Encryption initialized", "subtitle": "Flag to indicate if encryption is initialized", "expressions": [ "(( tikv_encryption_is_initialized {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Encryption meta files size", "subtitle": "Total size of encryption meta files", "expressions": [ "(( tikv_encryption_meta_file_size_bytes {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} ))" ] }, { "title": "Encrypt/decrypt data nanos", "subtitle": null, "expressions": [ "sum(rate( tikv_coprocessor_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"} [$__rate_interval] )) by (req, $additional_groupby)", "sum(rate( tikv_coprocessor_rocksdb_perf {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"} [$__rate_interval] )) by (req, $additional_groupby)" ] }, { "title": "Read/write encryption meta duration", "subtitle": "Writing or reading file duration (second)", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_encryption_write_read_file_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_encryption_write_read_file_duration_seconds_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_encryption_write_read_file_duration_seconds_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_encryption_write_read_file_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )" ] } ] }, { "title": "TTL", "panels": [ { "title": "TTL expire count", "subtitle": null, "expressions": [ "sum(rate( tikv_ttl_expire_kv_count_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "TTL expire size", "subtitle": null, "expressions": [ "sum(rate( tikv_ttl_expire_kv_size_total {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance)" ] }, { "title": "TTL check progress", "subtitle": null, "expressions": [ "(sum(rate( tikv_ttl_checker_processed_regions {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (instance) / sum(rate( tikv_raftstore_region_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"} [$__rate_interval] )) by (instance) )" ] }, { "title": "TTL checker actions", "subtitle": null, "expressions": [ "sum(rate( tikv_ttl_checker_actions {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (type, $additional_groupby)" ] }, { "title": "TTL checker compact duration", "subtitle": "The time consumed when executing GC tasks", "expressions": [ "histogram_quantile(0.9999,( sum(rate( tikv_ttl_checker_compact_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "histogram_quantile(0.99,( sum(rate( tikv_ttl_checker_compact_duration_bucket {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by (le, $additional_groupby) ))", "(sum(rate( tikv_ttl_checker_compact_duration_sum {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) / sum(rate( tikv_ttl_checker_compact_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby) )", "sum(rate( tikv_ttl_checker_compact_duration_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"} [$__rate_interval] )) by ($additional_groupby)" ] }, { "title": "TTL checker poll interval", "subtitle": null, "expressions": [ "max(( tikv_ttl_checker_poll_interval {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"} ))" ] } ] } ] }