{ "dashboard_title": "Serverless-Cluster-TiKV-Details", "rows": [ { "title": "Duration", "panels": [ { "title": "Write Pipeline Duration", "subtitle": "Write Duration Composition", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Cop Read Duration", "subtitle": "Read Duration Composition", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] } ] }, { "title": "Cluster", "panels": [ { "title": "Store size", "subtitle": "The storage size per TiKV instance", "expressions": [ "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"all_used\"}) by (instance)", "sum(tikv_store_size_bytes{instance=~\".*canary.*\", type=\"all_used\"})", "sum(tikv_store_size_bytes{instance=~\".*canary.*\", type=\"available\"})" ] }, { "title": "Available size", "subtitle": "The available capacity size of each TiKV instance", "expressions": [ "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"available\"}) by (instance)" ] }, { "title": "Capacity size", "subtitle": "The capacity size per TiKV instance", "expressions": [ "sum(tikv_store_size_bytes{instance=~\"$instance\", type=\"capacity\"}) by (instance)" ] }, { "title": "CPU", "subtitle": "The CPU usage of each TiKV instance", "expressions": [ "sum(rate(process_cpu_seconds_total{instance=~\"$instance\", job=~\".*tikv-serverless.*\"}[1m])) by (instance,job)" ] }, { "title": "Memory", "subtitle": "The memory usage per TiKV instance", "expressions": [ "avg(process_resident_memory_bytes{job=~\".*tikv.*\"}) by (instance)" ] }, { "title": "IO utilization", "subtitle": "The I/O utilization per TiKV instance", "expressions": [ "rate(node_disk_io_time_seconds_total{}[1m])" ] }, { "title": "MBps", "subtitle": "The total bytes of read and write in each TiKV instance", "expressions": [ "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[1m])) by (instance)", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[1m])) by (instance)" ] }, { "title": "QPS", "subtitle": "The QPS per command in each TiKV instance", "expressions": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance,type)" ] }, { "title": "Errps", "subtitle": "The total number of the gRPC message failures", "expressions": [ "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance)", "sum(delta(tikv_pd_heartbeat_message_total{instance=~\"$instance\", type=\"noop\"}[1m])) by (instance) < 1", "sum(rate(tikv_critical_error_total{instance=~\"$instance\"}[1m])) by (instance, type)" ] }, { "title": "Leader", "subtitle": " \tThe number of leaders on each TiKV instance", "expressions": [ "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}) by (instance)", "delta(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}[30s]) < -10" ] }, { "title": "Region", "subtitle": "The number of Regions and Buckets on each TiKV instance", "expressions": [ "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)", "sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"buckets\"}) by (instance)" ] }, { "title": "Uptime", "subtitle": "TiKV uptime since the last restart", "expressions": [ "(time() - process_start_time_seconds{job=~\".*tikv-serverless.*\"})" ] }, { "title": "Dfs Worker Healthy", "subtitle": null, "expressions": [ "raft_engine_dfs_worker_healthy{}" ] } ] }, { "title": "Errors", "panels": [ { "title": "Critical error", "subtitle": null, "expressions": [ "sum(rate(tikv_critical_error_total{instance=~\"$instance\"}[1m])) by (instance, type)" ] }, { "title": "Server is busy", "subtitle": "Indicates occurrences of events that make the TiKV instance unavailable temporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor Full", "expressions": [ "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\"}[1m])) by (instance)", "sum(rate(tikv_channel_full_total{instance=~\"$instance\"}[1m])) by (instance, type)", "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\", type='full'}[1m])) by (instance)", "avg(tikv_engine_write_stall{instance=~\"$instance\", type=\"write_stall_percentile99\", db=~\"$db\"}) by (instance, db)", "sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Server report failures", "subtitle": "The total number of reporting failure messages", "expressions": [ "sum(rate(tikv_server_report_failure_msg_total{instance=~\"$instance\"}[1m])) by (type,instance,store_id)" ] }, { "title": "Raftstore error", "subtitle": "The number of different raftstore errors on each TiKV instance", "expressions": [ "sum(rate(tikv_storage_engine_async_request_total{instance=~\"$instance\", status!~\"success|all\"}[1m])) by (instance, status)" ] }, { "title": "Scheduler error", "subtitle": "The number of scheduler errors per type on each TiKV instance", "expressions": [ "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\", stage=~\"snapshot_err|prepare_write_err\"}[1m])) by (instance, stage)" ] }, { "title": "Coprocessor error", "subtitle": "The number of different coprocessor errors on each TiKV instance", "expressions": [ "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (instance, reason)" ] }, { "title": "gRPC message error", "subtitle": "The number of gRPC message errors per type on each TiKV instance", "expressions": [ "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\"}[1m])) by (instance, type)" ] }, { "title": "Leader drop", "subtitle": "The count of dropped leaders per TiKV instance", "expressions": [ "sum(delta(tikv_raftstore_region_count{instance=~\"$instance\", type=\"leader\"}[1m])) by (instance)" ] }, { "title": "Leader missing", "subtitle": "The count of missing leaders per TiKV instance", "expressions": [ "sum(tikv_raftstore_leader_missing{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Damaged files", "subtitle": "RocksDB damaged SST files", "expressions": [ "tikv_rocksdb_damaged_files{}", "tikv_rocksdb_damaged_files_deleted{}" ] }, { "title": "Log Replication Rejected", "subtitle": "the count of Log Replication Reject caused by follower memory insufficient", "expressions": [ "sum(rate(tikv_server_raft_append_rejects{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (instance)" ] } ] }, { "title": "Server", "panels": [ { "title": "CF size", "subtitle": "The size of each column family", "expressions": [ "sum(tikv_engine_size_bytes{instance=~\"$instance\"}) by (type)" ] }, { "title": "Channel full", "subtitle": "The total number of channel full errors on each TiKV instance", "expressions": [ "sum(rate(tikv_channel_full_total{instance=~\"$instance\"}[1m])) by (instance, type)" ] }, { "title": "Active written leaders", "subtitle": "The number of leaders being written on each TiKV instance", "expressions": [ "sum(rate(tikv_region_written_keys_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Approximate Region size", "subtitle": "The approximate Region size", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_raftstore_region_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_raftstore_region_size_count{instance=~\"$instance\"}[1m])) " ] }, { "title": "Approximate Region size Histogram", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_region_size_bucket{instance=~\"$instance\"}[1m]))" ] }, { "title": "Region average written bytes", "subtitle": "The average rate of writing bytes to Regions per TiKV instance", "expressions": [ "sum(rate(tikv_region_written_bytes_sum{}[1m])) by (instance) / sum(rate(tikv_region_written_bytes_count{}[1m])) by (instance)" ] }, { "title": "Region written bytes", "subtitle": null, "expressions": [ "sum(rate(tikv_region_written_bytes_bucket[1m]))" ] }, { "title": "Region average written keys", "subtitle": "The average rate of written keys to Regions per TiKV instance", "expressions": [ "sum(rate(tikv_region_written_keys_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_region_written_keys_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Region written keys", "subtitle": null, "expressions": [ "sum(rate(tikv_region_written_keys_bucket{instance=~\"$instance\"}[1m]))" ] }, { "title": "Request batch input", "subtitle": "The size of requests into request batch per TiKV instance", "expressions": [ "sum(rate(tikv_server_request_batch_size_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_server_request_batch_size_count{instance=~\"$instance\"}[1m])) by (type)", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le, type))" ] }, { "title": "Hibernate Peers", "subtitle": " \tThe number of peers in hibernated state", "expressions": [ "sum(tikv_raftstore_hibernated_peer_state{instance=~\"$instance\"}) by (instance, state)" ] }, { "title": "Memory trace", "subtitle": null, "expressions": [ "tikv_server_mem_trace_sum{instance=~\"$instance\", name=~\"raftstore-.*\"}", "raft_engine_memory_usage{instance=~\"$instance\"}" ] }, { "title": "Raft Entry Cache Evicts", "subtitle": null, "expressions": [ "rate(tikv_raft_entries_evict_bytes{instance=~\"$instance\"}[1m])" ] }, { "title": "Resolve address duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_server_address_resolve_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "99% Thread Pool Schedule Wait Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\"}[1m])) by (le, name))" ] }, { "title": "Average Thread Pool Schedule Wait Duration", "subtitle": null, "expressions": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_sum{instance=~\"$instance\"}[1m])) by (name) / sum(rate(tikv_yatp_pool_schedule_wait_duration_count{instance=~\"$instance\"}[1m])) by (name)" ] }, { "title": "Disk IO time per second", "subtitle": null, "expressions": [ "sum(rate(tikv_storage_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_time\"}[1m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_time\"}[1m])) by (req)" ] }, { "title": "Disk IO bytes per second", "subtitle": null, "expressions": [ "sum(rate(tikv_storage_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_byte\"}[1m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\", metric=\"block_read_byte\"}[1m])) by (req)" ] } ] }, { "title": "gRPC", "panels": [ { "title": "gRPC message count", "subtitle": "The count of different kinds of gRPC message", "expressions": [ "sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)" ] }, { "title": "gRPC message failed", "subtitle": "The count of different kinds of gRPC message which is failed", "expressions": [ "sum(rate(tikv_grpc_msg_fail_total{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (type)" ] }, { "title": "Average gRPC message duration", "subtitle": null, "expressions": [ "sum(rate(tikv_grpc_msg_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "gRPC batch size", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_server_grpc_req_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_grpc_req_batch_size_count{instance=~\"$instance\"}[1m]))", "sum(rate(tikv_server_grpc_resp_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_server_request_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_request_batch_size_count{instance=~\"$instance\"}[1m]))" ] }, { "title": "99% gRPC message duration", "subtitle": "The execution time of gRPC message", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))" ] }, { "title": "raft message batch size", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_server_raft_message_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_server_raft_message_batch_size_count{instance=~\"$instance\"}[1m]))" ] }, { "title": "9999% gRPC message duration", "subtitle": "The execution time of gRPC message", "expressions": [ "histogram_quantile(0.9999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))" ] }, { "title": "gRPC request sources duration", "subtitle": "The duration of different sources of gRPC request", "expressions": [ "sum(rate(tikv_grpc_request_source_duration_vec{instance=~\"$instance\"}[1m])) by (source)" ] }, { "title": "9999999999% gRPC message duration", "subtitle": "The execution time of gRPC message", "expressions": [ "histogram_quantile(0.9999999999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\"$instance\", type!=\"kv_gc\"}[1m])) by (le, type))" ] }, { "title": "gRPC request sources QPS", "subtitle": "The QPS of different sources of gRPC request", "expressions": [ "sum(rate(tikv_grpc_request_source_counter_vec{instance=~\"$instance\"}[1m])) by (source)" ] } ] }, { "title": "Thread CPU", "panels": [ { "title": "Raft store CPU", "subtitle": "The CPU utilization of raftstore thread", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)" ] }, { "title": "Async apply CPU", "subtitle": "The CPU utilization of async apply", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_[0-9]+\"}[1m])) by (instance)" ] }, { "title": "Store writer CPU", "subtitle": "The CPU utilization of store writer thread", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_write.*\"}[1m])) by (instance)" ] }, { "title": "gRPC poll CPU", "subtitle": "The CPU utilization of gRPC", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"grpc.*\"}[1m])) by (instance)" ] }, { "title": "Scheduler worker CPU", "subtitle": "The CPU utilization of scheduler worker", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sched_.*\"}[1m])) by (instance)" ] }, { "title": "Storage ReadPool CPU", "subtitle": "The CPU utilization of readpool", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_norm.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_high.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"store_read_low.*\"}[1m])) by (instance)" ] }, { "title": "Unified read pool CPU", "subtitle": "The CPU utilization of the unified read pool", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"unified_read_po.*\"}[1m])) by (instance)" ] }, { "title": "RocksDB CPU", "subtitle": "The CPU utilization of RocksDB", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)" ] }, { "title": "Coprocessor CPU", "subtitle": "The CPU utilization of coprocessor", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_normal.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_high.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cop_low.*\"}[1m])) by (instance)" ] }, { "title": "GC worker CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"gc_worker.*\"}[1m])) by (instance)" ] }, { "title": "BackGround Worker CPU", "subtitle": " \tThe CPU utilization of split check", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"background.*\"}[1m])) by (instance)" ] }, { "title": "Import CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)" ] }, { "title": "Backup Worker CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_endpoint\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance, tid) > 0", "sum(tikv_backup_thread_pool_size{instance=~\"$instance\"}) by(instance)" ] }, { "title": "CDC worker CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"tso\"}[1m])) by (instance)" ] }, { "title": "CDC endpoint CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"cdc_.*\"}[1m])) by (instance)" ] }, { "title": "Raftlog fetch Worker CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftlog_fetch.*\"}[1m])) by (instance)" ] }, { "title": "TSO Worker CPU", "subtitle": "The CPU utilization of TSO worker", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"tso_worker\"}[1m])) by (instance)" ] } ] }, { "title": "TTL", "panels": [ { "title": "TTL check progress", "subtitle": null, "expressions": [ "sum(tikv_ttl_checker_processed_regions{instance=~\"$instance\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)" ] }, { "title": "TTL checker actions", "subtitle": null, "expressions": [ "sum(rate(tikv_ttl_checker_actions{instance=~\"$instance\"}[30s])) by (type)" ] }, { "title": "TTL checker compact duration", "subtitle": "The time consumed when executing GC tasks", "expressions": [ "histogram_quantile(1, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_ttl_checker_compact_duration_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_ttl_checker_compact_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_ttl_checker_compact_duration_count{instance=~\"$instance\"}[1m]))" ] }, { "title": "TTL checker poll interval", "subtitle": null, "expressions": [ "max(tikv_ttl_checker_poll_interval{instance=~\"$instance\"})" ] } ] }, { "title": "PD", "panels": [ { "title": "PD requests", "subtitle": "The count of requests that TiKV sends to PD", "expressions": [ "sum(rate(tikv_pd_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "PD request duration (average)", "subtitle": "The time consumed by requests that TiKV sends to PD", "expressions": [ "sum(rate(tikv_pd_request_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "PD heartbeats", "subtitle": " \tThe total number of PD heartbeat messages", "expressions": [ "sum(rate(tikv_pd_heartbeat_message_total{instance=~\"$instance\"}[1m])) by (type)", "tikv_pd_pending_heartbeat_total{instance=~\"$instance\"}" ] }, { "title": "PD validate peers", "subtitle": "The total number of peers validated by the PD worker", "expressions": [ "sum(rate(tikv_pd_validate_peer_total{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "PD reconnections", "subtitle": " \tThe count of reconnections between TiKV and PD", "expressions": [ "sum(delta(tikv_pd_reconnect_total{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "PD forward status", "subtitle": " \tThe forward status of PD client", "expressions": [ "tikv_pd_request_forwarded{instance=~\"$instance\"}" ] }, { "title": "Pending TSO Requests", "subtitle": "The number of TSO requests waiting in the queue.", "expressions": [ "tikv_pd_pending_tso_request_total{instance=~\"$instance\"}" ] }, { "title": "Store Slow Score", "subtitle": "The slow score of stores", "expressions": [ "tikv_raftstore_slow_score{instance=~\"$instance\"}" ] }, { "title": "Inspected duration per server", "subtitle": "The duration that recorded by inspecting messages.", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_inspect_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance, type))" ] } ] }, { "title": "IO Breakdown", "panels": [ { "title": "Write IO bytes", "subtitle": "The throughput of disk write per IO type", "expressions": [ "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"write\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"write\"}[45s]))" ] }, { "title": "Read IO bytes", "subtitle": "The throughput of disk read per IO type", "expressions": [ "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"read\"}[45s])) by (type)", "sum(rate(tikv_io_bytes{instance=~\"$instance\", op=\"read\"}[45s]))" ] }, { "title": "IO threshold", "subtitle": "The threshold of disk IOs per priority", "expressions": [ "avg(tikv_rate_limiter_max_bytes_per_sec{instance=~\"$instance\"}) by (type)" ] }, { "title": "Rate Limiter Request Wait Duration", "subtitle": "IO rate limiter request wait duration.", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le, type))", "rate(tikv_rate_limiter_request_wait_duration_seconds_sum{instance=~\"$instance\"}[30s]) / rate(tikv_rate_limiter_request_wait_duration_seconds_count{instance=~\"$instance\"}[30s])" ] } ] }, { "title": "Raft Waterfall", "panels": [ { "title": "Storage async write duration", "subtitle": "The time consumed by processing asynchronous write requests", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[30s])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[30s])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[30s]))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[30s]))" ] }, { "title": "Store duration", "subtitle": "The store time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_duration_secs_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Apply duration", "subtitle": "The apply time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_apply_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_apply_duration_secs_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store propose wait duration", "subtitle": "The propose wait time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store batch wait duration", "subtitle": "The batch wait time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_batch_wait_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store send to write queue duration", "subtitle": "The send-to-write-queue time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_send_to_queue_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store send proposal duration", "subtitle": "The send raft message of the proposal duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_send_proposal_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store write kv db end duration", "subtitle": "The write kv db end duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store before write duration", "subtitle": "The before write time duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_before_write_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store persist duration", "subtitle": "The persist duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_persist_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store write end duration", "subtitle": "The write end duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_write_end_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store commit but not persist duration", "subtitle": "The commit but not persist duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] }, { "title": "Store commit and persist duration", "subtitle": "The commit and persist duration of each request", "expressions": [ "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_sum{instance=~\"$instance\"}[30s])) / sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[30s]))", "sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_count{instance=~\"$instance\"}[30s]))" ] } ] }, { "title": "Raft IO", "panels": [ { "title": "Process ready duration", "subtitle": "The time consumed for peer processes to be ready in Raft", "expressions": [ "sum(delta(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\"$instance\", type='ready'}[1m])) by (le)" ] }, { "title": "99% Process ready duration per server", "subtitle": "The time consumed for peer processes to be ready in Raft", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\"$instance\", type='ready'}[1m])) by (le, instance))" ] }, { "title": "Store write loop duration", "subtitle": "The time duration of store write loop when store-io-pool-size is not zero.", "expressions": [ "sum(delta(tikv_raftstore_store_write_loop_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Store write loop duration per server", "subtitle": "The time duration of store write loop on each TiKV instance when store-io-pool-size is not zero.", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_loop_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Append log duration", "subtitle": "The time consumed when Raft appends log", "expressions": [ "sum(delta(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Append log duration per server", "subtitle": "The time consumed when Raft appends log on each TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_kvdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_send_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_callback_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Commit log duration", "subtitle": "The time consumed when Raft commits log", "expressions": [ "sum(delta(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Commit log duration per server", "subtitle": "The time consumed when Raft commits log on each TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Apply log duration", "subtitle": "The time consumed when Raft applies log", "expressions": [ "sum(delta(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Apply log duration per server", "subtitle": "The time consumed for Raft to apply logs per TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Store io task reschedule", "subtitle": null, "expressions": [ "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", "sum(tikv_raftstore_io_reschedule_pending_tasks_total{instance=~\"$instance\"}) by (instance)" ] }, { "title": "99% Write task block duration per server", "subtitle": "The time consumed when store write task block on each TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] } ] }, { "title": "Raft Propose", "panels": [ { "title": "Raft proposals per ready", "subtitle": "The proposal count of all Regions in a mio tick", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Raft read/write proposals", "subtitle": "The number of proposals per type", "expressions": [ "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"local_read|normal|read_index\"}[1m])) by (type)" ] }, { "title": "Raft read proposals per server", "subtitle": "The number of read proposals which are made by each TiKV instance", "expressions": [ "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"local_read|read_index\"}[1m])) by (instance)" ] }, { "title": "Raft write proposals per server", "subtitle": "The number of write proposals which are made by each TiKV instance", "expressions": [ "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=\"normal\"}[1m])) by (instance)" ] }, { "title": "Propose wait duration", "subtitle": "The wait time of each proposal", "expressions": [ "sum(delta(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Propose wait duration per server", "subtitle": "The wait time of each proposal in each TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Store write wait duration", "subtitle": "The wait time of each store write task", "expressions": [ "sum(delta(tikv_raftstore_store_write_task_wait_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Store write wait duration per server", "subtitle": "The wait time of each store write task in each TiKV instance", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Apply wait duration", "subtitle": null, "expressions": [ "sum(delta(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "99% Apply wait duration per server", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Store write handle msg duration", "subtitle": "The handle duration of each store write task msg", "expressions": [ "sum(delta(tikv_raftstore_store_write_handle_msg_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Store write trigger size", "subtitle": "The distribution of write trigger size", "expressions": [ "sum(delta(tikv_raftstore_store_write_trigger_wb_bytes_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Raft propose speed", "subtitle": "The rate at which peers propose logs", "expressions": [ "avg(rate(tikv_raftstore_propose_log_size_sum{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Perf Context duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, type))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, type))" ] } ] }, { "title": "Raft Process", "panels": [ { "title": "Ready handled", "subtitle": "The count of different ready type of Raft", "expressions": [ "sum(rate(tikv_raftstore_raft_ready_handled_total{instance=~\"$instance\"}[1m])) by (type)", "sum(rate(tikv_raftstore_raft_process_duration_secs_count{instance=~\"$instance\", type=\"ready\"}[1m]))" ] }, { "title": "Max duration of raft store events", "subtitle": "The max time consumed by raftstore events", "expressions": [ "histogram_quantile(1.0, sum(rate(tikv_raftstore_event_duration_bucket{instance=~\"$instance\"}[1m])) by (le, type))", "histogram_quantile(1.0, sum(rate(tikv_broadcast_normal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Replica read lock checking duration", "subtitle": "The time consumed for checking memory locks for replica reads", "expressions": [ "sum(delta(tikv_replica_read_lock_check_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Peer msg length distribution", "subtitle": "The length of peer msgs for each round handling", "expressions": [ "sum(delta(tikv_raftstore_peer_msg_len_bucket{instance=~\"$instance\"}[1m])) by (le)" ] } ] }, { "title": "Raft Message", "panels": [ { "title": "Sent messages per server", "subtitle": "The number of Raft messages sent by each TiKV instance", "expressions": [ "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Flush messages per server", "subtitle": "The number of Raft messages flushed by each TiKV instance", "expressions": [ "sum(rate(tikv_server_raft_message_flush_total{instance=~\"$instance\"}[1m])) by (instance, reason)" ] }, { "title": "Receive messages per server", "subtitle": "The number of Raft messages received by each TiKV instance", "expressions": [ "sum(rate(tikv_server_raft_message_recv_total{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Messages", "subtitle": "The number of different types of Raft messages that are sent", "expressions": [ "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Vote", "subtitle": "The total number of vote messages that are sent in Raft", "expressions": [ "sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"vote\"}[1m])) by (instance)" ] }, { "title": "Raft dropped messages", "subtitle": "The number of dropped Raft messages per type", "expressions": [ "sum(rate(tikv_raftstore_raft_dropped_message_total{instance=~\"$instance\"}[1m])) by (type)" ] } ] }, { "title": "Raft Admin", "panels": [ { "title": "Admin proposals", "subtitle": "The number of admin proposals", "expressions": [ "sum(rate(tikv_raftstore_proposal_total{instance=~\"$instance\", type=~\"conf_change|transfer_leader\"}[1m])) by (type)" ] }, { "title": "Admin apply", "subtitle": "The number of the processed apply command", "expressions": [ "sum(rate(tikv_raftstore_admin_cmd_total{instance=~\"$instance\", status=\"success\", type!=\"compact\"}[1m])) by (type)" ] }, { "title": "Check split", "subtitle": "The number of raftstore split checksss", "expressions": [ "sum(rate(tikv_raftstore_check_split_total{instance=~\"$instance\", type!=\"ignore\"}[1m])) by (type)" ] }, { "title": "99.99% Check split duration", "subtitle": "The time consumed when running split check in .9999", "expressions": [ "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Load base split event", "subtitle": null, "expressions": [ "sum(delta(tikv_load_base_split_event[1m])) by (type)" ] }, { "title": "TopN QPS exceeds threshold", "subtitle": null, "expressions": [ "tikv_read_qps_topn{order=\"0\"}" ] }, { "title": "Load base split duration", "subtitle": null, "expressions": [ "histogram_quantile(0.80, sum(rate(tikv_load_base_split_duration_seconds_bucket{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.90, sum(rate(tikv_load_base_split_duration_seconds_bucket{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (le, instance))", "sum(rate(tikv_load_base_split_duration_seconds_sum{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}[1m])) by (instance)" ] } ] }, { "title": "Raft Log", "panels": [ { "title": "Raft log GC write duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC kv sync duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_kv_sync_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC write operations ", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_raft_log_gc_write_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC deleted keys", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_sum{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC seek operations ", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_raft_log_gc_seek_operations_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC write batch size", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_deleted_keys_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log GC purge duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_sum{instance=~\"$instance\"}[1m])) by (instance) / sum(rate(tikv_raftstore_raft_log_gc_purge_duration_secs_count{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log lag", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_log_lag_sum{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log gc skipped", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_raft_log_gc_skipped{instance=~\"$instance\"}[1m])) by (instance, reason)" ] }, { "title": "Raft log GC failed", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_raft_log_gc_failed{instance=~\"$instance\"}[1m])) by (instance)" ] }, { "title": "Raft log fetch", "subtitle": null, "expressions": [ "sum(rate(tikv_raftstore_entry_fetches{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Raft log async fetch task duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_raftstore_entry_fetches_task_duration_seconds_count{instance=~\"$instance\"}[1m]))", "sum(tikv_worker_pending_task_total{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\", name=~\"raftlog-fetch-worker\"})" ] } ] }, { "title": "Local reader", "panels": [ { "title": "Local reader requests", "subtitle": "The number of rejections from the local read thread and The number of total requests", "expressions": [ "sum(rate(tikv_raftstore_local_read_reject_total{instance=~\"$instance\"}[1m])) by (instance, reason)", "sum(rate(tikv_raftstore_local_read_executed_requests{instance=~\"$instance\"}[1m])) by (instance)", "sum(rate(tikv_raftstore_local_read_executed_stale_read_requests{instance=~\"$instance\"}[1m])) by (instance)" ] } ] }, { "title": "Unified Read Pool", "panels": [ { "title": "Time used by level", "subtitle": "The time used by each level in the unified read pool per second. Level 0 refers to small queries.", "expressions": [ "sum(rate(tikv_multilevel_level_elapsed{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (level)" ] }, { "title": "Level 0 chance", "subtitle": "The chance that level 0 (small) tasks are scheduled in the unified read pool.", "expressions": [ "tikv_multilevel_level0_chance{instance=~\"$instance\", name=\"unified-read-pool\"}" ] }, { "title": "Running tasks", "subtitle": "The number of concurrently running tasks in the unified read pool.", "expressions": [ "sum(avg_over_time(tikv_unified_read_pool_running_tasks{}[1m])) by (instance)" ] }, { "title": "Unified Read Pool Wait Duration", "subtitle": null, "expressions": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\", name=~\"unified-read.*\"}[1m])) by (le)" ] }, { "title": "Duration of One Time Slice", "subtitle": "Unified read pool task execution time during one schedule.", "expressions": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_poll_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))" ] }, { "title": "Task Execute Duration", "subtitle": "Unified read pool task total execution duration.", "expressions": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_exec_duration_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))" ] }, { "title": "Task Schedule Times", "subtitle": "Task schedule number of times.", "expressions": [ "histogram_quantile(0.50, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(tikv_yatp_task_execute_times_bucket{instance=~\"$instance\", name=\"unified-read-pool\"}[1m])) by (le))" ] } ] }, { "title": "Storage", "panels": [ { "title": "Storage command total", "subtitle": "The total count of different kinds of commands received", "expressions": [ "sum(rate(tikv_storage_command_total{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Storage async request error", "subtitle": "The total number of engine asynchronous request errors", "expressions": [ "sum(rate(tikv_storage_engine_async_request_total{instance=~\"$instance\", status!~\"all|success\"}[1m])) by (status)" ] }, { "title": "Storage async snapshot duration", "subtitle": "The time consumed by processing asynchronous snapshot requests", "expressions": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)" ] }, { "title": "Storage async write duration", "subtitle": "The time consumed by processing asynchronous write requests", "expressions": [ "sum(delta(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le)" ] }, { "title": "Storage async snapshot duration", "subtitle": "The storage async snapshot duration", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"snapshot\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"snapshot\"}[1m]))" ] }, { "title": "Storage async write duration", "subtitle": "The storage async write duration", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\"$instance\", type=\"write\"}[1m])) by (le))", "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\"$instance\", type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\"$instance\", type=\"write\"}[1m]))" ] } ] }, { "title": "Resource Control", "panels": [ { "title": "Thread CPU Usage", "subtitle": "The CPU utilization of the unified read pool", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"raft_io.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"apply_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"status_server.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"pd_worker_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"grpc_server_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"unified_read_pool_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"remote_coproces.*\"}[1m])) by (instance)" ] }, { "title": "Request wait count", "subtitle": "The count of different kinds of request wait", "expressions": [ "sum(rate(tikv_request_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance,type)" ] }, { "title": "Keyspace read MBps", "subtitle": "The total bytes of read and write in each active keyspace", "expressions": [ "sum(rate(tikv_active_keyspace_read_bytes{instance=~\"$instance\"}[1m])) by (instance,keyspace_id)" ] }, { "title": "Average request wait duration", "subtitle": "The wait time of request", "expressions": [ "sum(rate(tikv_request_wait_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (instance,type) / sum(rate(tikv_request_wait_duration_seconds_count{instance=~\"$instance\"}[1m])) by (instance,type)" ] }, { "title": "99% request wait duration", "subtitle": "The wait time of request", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,type))" ] }, { "title": "9999% request wait duration", "subtitle": "The wait time of request", "expressions": [ "histogram_quantile(0.9999, sum(rate(tikv_request_wait_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,type))" ] } ] }, { "title": "Flow Control", "panels": [ { "title": "Scheduler flow", "subtitle": null, "expressions": [ "sum(tikv_scheduler_write_flow{instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_throttle_flow{instance=~\"$instance\"}) by (instance) != 0" ] }, { "title": "Scheduler discard ratio", "subtitle": null, "expressions": [ "sum(tikv_scheduler_discard_ratio{instance=~\"$instance\"}) by (instance) / 10000000" ] }, { "title": "Throttle duration", "subtitle": null, "expressions": [ "sum(delta(tikv_scheduler_throttle_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Compaction pending bytes", "subtitle": null, "expressions": [ "sum(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"kv\"}) by (cf)", "max(tikv_scheduler_pending_compaction_bytes{instance=~\"$instance\"}) by (instance) / 10000000" ] }, { "title": "Flow controller actions", "subtitle": null, "expressions": [ "sum(rate(tikv_scheduler_throttle_action_total{instance=~\"$instance\"}[1m])) by (cf, type)", "sum(rate(kv_engine_throttle_action_total{instance=~\"$instance\"}[1m])) by (limiter, action)", "sum(rate(kv_engine_limiter_throttle_counter{instance=~\"$instance\"}[1m])) by (limiter)" ] }, { "title": "Flow controller factors", "subtitle": null, "expressions": [ "max(tikv_scheduler_l0{instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_memtable{instance=~\"$instance\"}) by (instance)", "max(tikv_scheduler_l0_avg{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Txn command throttled duration", "subtitle": "Throttle time for txn storage commands in 1 minute.", "expressions": [ "sum(rate(tikv_txn_command_throttle_time_total{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Non-txn command throttled duration", "subtitle": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", "expressions": [ "sum(rate(tikv_non_txn_command_throttle_time_total{instance=~\"$instance\"}[1m])) by (type) " ] }, { "title": "Scheduler throttled CF", "subtitle": "The count of pending commands per TiKV instance", "expressions": [ "tikv_scheduler_throttle_cf{instance=~\"$instance\"} != 0" ] }, { "title": "Flush/L0 flow", "subtitle": null, "expressions": [ "sum(tikv_scheduler_l0_flow{instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_flush_flow{instance=~\"$instance\"}) by (instance, cf)", "sum(tikv_scheduler_l0_flow{instance=~\"$instance\"}) by (instance)", "sum(tikv_scheduler_flush_flow{instance=~\"$instance\"}) by (instance)" ] } ] }, { "title": "Scheduler - $command", "panels": [ { "title": "Scheduler stage total", "subtitle": "The total number of commands on each stage in commit command", "expressions": [ "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\", type=\"$command\"}[1m]))", "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\", type=\"$command\"}[1m])) by (stage)" ] }, { "title": "Scheduler command duration", "subtitle": "The time consumed when executing commit command", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "sum(rate(tikv_scheduler_command_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m])) ", "histogram_quantile(0.9999999999, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))" ] }, { "title": "Scheduler latch wait duration", "subtitle": "The time which is caused by latch wait in commit command", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m])) ", "histogram_quantile(0.9999, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.99999999, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))" ] }, { "title": "Scheduler keys read", "subtitle": "The count of keys read by a commit command", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_read_sum{instance=~\"$instance\", type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{instance=~\"$instance\", type=\"$command\"}[1m])) " ] }, { "title": "Scheduler keys written", "subtitle": "The count of keys written by a commit command", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "sum(rate(tikv_scheduler_kv_command_key_write_sum{instance=~\"$instance\", type=\"$command\"}[1m])) by (type) / sum(rate(tikv_scheduler_kv_command_key_write_count{instance=~\"$instance\", type=\"$command\"}[1m]))" ] }, { "title": "Scheduler scan details", "subtitle": "The keys scan details of each CF when executing commit command", "expressions": [ "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", req=\"$command\"}[1m])) by (tag)" ] }, { "title": "Scheduler scan details [lock]", "subtitle": "The keys scan details of lock CF when executing commit command", "expressions": [ "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"lock\", req=\"$command\"}[1m])) by (tag)" ] }, { "title": "Scheduler scan details [write]", "subtitle": "The keys scan details of write CF when executing commit command", "expressions": [ "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"write\", req=\"$command\"}[1m])) by (tag)" ] }, { "title": "Scheduler scan details [default]", "subtitle": "The keys scan details of default CF when executing commit command", "expressions": [ "sum(rate(tikv_scheduler_kv_scan_details{instance=~\"$instance\", cf=\"default\"}[1m])) by (tag,req)" ] }, { "title": "Scheduler command read duration", "subtitle": "The time consumed on reading when executing commit command", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\"$instance\", type=\"$command\"}[1m])) by (le))", "sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{instance=~\"$instance\", type=\"$command\"}[1m])) by (type) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{instance=~\"$instance\", type=\"$command\"}[1m]))" ] } ] }, { "title": "Scheduler", "panels": [ { "title": "Scheduler command count", "subtitle": "The total number of commands on each stage", "expressions": [ "sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Scheduler command duration", "subtitle": "The total number of commands on each stage", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\"$instance\",}[1m])) by (le, type))" ] }, { "title": "Scheduler stage total", "subtitle": "The total number of commands on each stage", "expressions": [ "sum(rate(tikv_scheduler_too_busy_total{instance=~\"$instance\"}[1m])) by (stage)", "sum(rate(tikv_scheduler_stage_total{instance=~\"$instance\"}[1m])) by (stage)" ] }, { "title": "Scheduler writing bytes", "subtitle": "The total writing bytes of commands on each stage", "expressions": [ "sum(tikv_scheduler_writing_bytes{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Scheduler priority commands", "subtitle": "The count of different priority commands", "expressions": [ "sum(rate(tikv_scheduler_commands_pri_total{instance=~\"$instance\"}[1m])) by (priority)" ] }, { "title": "Scheduler pending commands", "subtitle": "The count of pending commands per TiKV instance", "expressions": [ "sum(tikv_scheduler_contex_total{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Txn Scheduler Pool Wait Duration", "subtitle": null, "expressions": [ "sum(rate(tikv_yatp_pool_schedule_wait_duration_bucket{instance=~\"$instance\", name=~\"sched-worker.*\"}[1m])) by (le)" ] } ] }, { "title": "GC", "panels": [ { "title": "GC tasks", "subtitle": "The count of GC tasks processed by gc_worker", "expressions": [ "sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\"$instance\"}[1m])) by (task)", "sum(rate(tikv_storage_gc_skipped_counter{instance=~\"$instance\"}[1m])) by (task)", "sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\"$instance\"}[1m])) by (task)", "sum(rate(tikv_gc_worker_too_busy{instance=~\"$instance\"}[1m]))" ] }, { "title": "GC tasks duration", "subtitle": "The time consumed when executing GC tasks", "expressions": [ "histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", "histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", "histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\"$instance\"}[1m])) by (le, task))", "sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\"$instance\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\"$instance\"}[1m])) by (task)" ] }, { "title": "TiDB GC seconds", "subtitle": "The GC duration", "expressions": [ "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket{}[1m])) by (instance, le))" ] }, { "title": "TiDB GC worker actions", "subtitle": "The count of TiDB GC worker actions", "expressions": [ "sum(rate(tidb_tikvclient_gc_worker_actions_total{}[1m])) by (type)" ] }, { "title": "ResolveLocks Progress", "subtitle": "Progress of ResolveLocks, the first phase of GC", "expressions": [ "max(tidb_tikvclient_range_task_stats{type=~\"resolve-locks.*\"}) by (result)" ] }, { "title": "TiKV Auto GC Progress", "subtitle": "Progress of TiKV's GC", "expressions": [ "sum(tikv_gcworker_autogc_processed_regions{instance=~\"$instance\", type=\"scan\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\"$instance\", type=\"region\"}) by (instance)" ] }, { "title": "GC speed", "subtitle": "keys / second", "expressions": [ "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum{instance=~\"$instance\"}[1m])) by (key_mode)" ] }, { "title": "TiKV Auto GC SafePoint", "subtitle": "SafePoint used for TiKV's Auto GC", "expressions": [ "max(tikv_gcworker_autogc_safe_point{}) by (instance) / (2^18)" ] }, { "title": "GC lifetime", "subtitle": " \tThe lifetime of TiDB GC", "expressions": [ "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})" ] }, { "title": "GC interval", "subtitle": "The interval of TiDB GC", "expressions": [ "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})" ] }, { "title": "GC in Compaction Filter", "subtitle": "Keys handled in GC compaction filter", "expressions": [ "sum(rate(tikv_gc_compaction_filtered{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_skip{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_mvcc_rollback{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_orphan_versions{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_perform{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_failure{instance=~\"$instance\"}[1m])) by (key_mode,type)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_met{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_handled{instance=~\"$instance\"}[1m])) by (key_mode)", "sum(rate(tikv_gc_compaction_filter_mvcc_deletion_wasted{instance=~\"$instance\"}[1m])) by (key_mode)" ] }, { "title": "GC scan write details", "subtitle": "GC scan write details", "expressions": [ "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"write\"}[1m])) by (key_mode,tag)" ] }, { "title": "GC scan default details", "subtitle": "GC scan default details", "expressions": [ "sum(rate(tikv_gcworker_gc_keys{instance=~\"$instance\", cf=\"default\"}[1m])) by (key_mode,tag)" ] } ] }, { "title": "Snapshot", "panels": [ { "title": "Rate snapshot message", "subtitle": "The rate of Raft snapshot messages sent", "expressions": [ "sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\"$instance\", type=\"snapshot\"}[1m]))" ] }, { "title": "99% Handle snapshot duration", "subtitle": "The time consumed when handling snapshots", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"apply\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\"$instance\", type=\"generate\"}[1m])) by (le))" ] }, { "title": "Snapshot state count", "subtitle": "The number of snapshots in different states", "expressions": [ "sum(tikv_raftstore_snapshot_traffic_total{instance=~\"$instance\"}) by (type)" ] }, { "title": "99.99% Snapshot size", "subtitle": "The snapshot size (P99.99).9999", "expressions": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "99.99% Snapshot KV count", "subtitle": "The number of KV within a snapshot in .9999", "expressions": [ "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Snapshot Actions", "subtitle": "Action stats for snapshot generating and applying", "expressions": [ "sum(delta(tikv_raftstore_snapshot_total{instance=~\"$instance\"}[1m])) by (type, status)", "sum(delta(tikv_raftstore_clean_region_count{instance=~\"$instance\"}[1m])) by (type)" ] } ] }, { "title": "Task", "panels": [ { "title": "Worker handled tasks", "subtitle": "The number of tasks handled by worker", "expressions": [ "sum(rate(tikv_worker_handled_task_total{instance=~\"$instance\"}[1m])) by (name)" ] }, { "title": "Worker pending tasks", "subtitle": " \tCurrent pending and running tasks of worker", "expressions": [ "sum(tikv_worker_pending_task_total{instance=~\"$instance\"}) by (name)" ] }, { "title": "FuturePool handled tasks", "subtitle": "The number of tasks handled by future_pool", "expressions": [ "sum(rate(tikv_futurepool_handled_task_total{instance=~\"$instance\"}[1m])) by (name)" ] }, { "title": "FuturePool pending tasks", "subtitle": "Current pending and running tasks of future_pool", "expressions": [ "sum(rate(tikv_futurepool_pending_task_total{instance=~\"$instance\"}[1m])) by (name)" ] } ] }, { "title": "Coprocessor Overview", "panels": [ { "title": "Request duration", "subtitle": "The time consumed to handle coprocessor read requests", "expressions": [ "sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Request duration", "subtitle": null, "expressions": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))" ] }, { "title": "Total Requests", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (req)" ] }, { "title": "Total Request Errors", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_request_error{instance=~\"$instance\"}[1m])) by (reason)" ] }, { "title": "Total KV Cursor Operations", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\"$instance\"}[1m])) by (req)" ] }, { "title": "KV Cursor Operations", "subtitle": null, "expressions": [ "histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) ", "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\"$instance\"}[1m])) by (le, req)) " ] }, { "title": "Total RocksDB Perf Statistics", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m])) by (req)" ] }, { "title": "Total Response Size", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_response_bytes{instance=~\"$instance\"}[1m]))" ] } ] }, { "title": "Coprocessor Detail", "panels": [ { "title": "Handle duration", "subtitle": "The time consumed when handling coprocessor requests", "expressions": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))" ] }, { "title": "95% Handle duration by store", "subtitle": "The time consumed to handle coprocessor requests per TiKV instance (P95)", "expressions": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))" ] }, { "title": "Wait duration", "subtitle": "The time consumed when coprocessor requests are wait for being handled", "expressions": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,req))" ] }, { "title": "95% Wait duration by store", "subtitle": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "expressions": [ "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance,req))" ] }, { "title": "Total DAG Requests", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_dag_request_count{instance=~\"$instance\"}[1m])) by (vec_type)" ] }, { "title": "Total DAG Executors", "subtitle": "The total number of DAG executors", "expressions": [ "sum(rate(tikv_coprocessor_executor_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Total Ops Details (Table Scan)", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag)" ] }, { "title": "Total Ops Details (Index Scan)", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag)" ] }, { "title": "Total Ops Details by CF (Table Scan)", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"select\"}[1m])) by (tag,cf)" ] }, { "title": "Total Ops Details by CF (Index Scan)", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_scan_details{instance=~\"$instance\", req=\"index\"}[1m])) by (tag,cf)" ] }, { "title": "Memory lock checking duration", "subtitle": "The time consumed on checking memory locks for coprocessor requests", "expressions": [ "sum(rate(tikv_coprocessor_mem_lock_check_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] } ] }, { "title": "Threads", "panels": [ { "title": "Threads state", "subtitle": null, "expressions": [ "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance, state)", "sum(tikv_threads_state{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Threads IO", "subtitle": null, "expressions": [ "topk(20, sum(rate(tikv_threads_io_bytes_total{instance=~\"$instance\"}[30s])) by (name, io) > 1024)" ] }, { "title": "Thread Voluntary Context Switches", "subtitle": null, "expressions": [ "topk(20, max(rate(tikv_thread_voluntary_context_switches{instance=~\"$instance\"}[30s])) by (name) > 200)" ] }, { "title": "Thread Nonvoluntary Context Switches", "subtitle": null, "expressions": [ "topk(20, max(rate(tikv_thread_nonvoluntary_context_switches{instance=~\"$instance\"}[30s])) by (name) > 100)" ] } ] }, { "title": "RocksDB - $db", "panels": [ { "title": "Get operations", "subtitle": "The count of get operations", "expressions": [ "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l0\"}[1m]))", "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l1\"}[1m]))", "sum(rate(tikv_engine_get_served{instance=~\"$instance\", db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))" ] }, { "title": "Get duration", "subtitle": "The time consumed when executing get operations", "expressions": [ "max(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_max\"})", "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile99\"})", "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_percentile95\"})", "avg(tikv_engine_get_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"get_average\"})" ] }, { "title": "Seek operations", "subtitle": "The count of seek operations", "expressions": [ "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek\"}[1m]))", "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_seek_found\"}[1m]))", "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next\"}[1m]))", "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_next_found\"}[1m]))", "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev\"}[1m]))", "sum(rate(tikv_engine_locate{instance=~\"$instance\", db=\"$db\", type=\"number_db_prev_found\"}[1m]))" ] }, { "title": "Seek duration", "subtitle": "The time consumed when executing seek operation", "expressions": [ "max(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_max\"})", "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile99\"})", "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_percentile95\"})", "avg(tikv_engine_seek_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"seek_average\"})" ] }, { "title": "Write operations", "subtitle": "The count of write operations", "expressions": [ "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_timeout\"}[1m]))", "sum(rate(tikv_engine_write_served{instance=~\"$instance\", db=\"$db\", type=\"write_with_wal\"}[1m]))" ] }, { "title": "Write duration", "subtitle": "The time consumed when executing write operation", "expressions": [ "max(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_max\"})", "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile99\"})", "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_percentile95\"})", "avg(tikv_engine_write_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_average\"})" ] }, { "title": "WAL sync operations", "subtitle": " \tThe count of WAL sync operations", "expressions": [ "sum(rate(tikv_engine_wal_file_synced{instance=~\"$instance\", db=\"$db\"}[1m]))" ] }, { "title": "Write WAL duration", "subtitle": "The time consumed when executing write wal operation", "expressions": [ "max(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_max\"})", "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile99\"})", "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_percentile95\"})", "avg(tikv_engine_write_wal_time_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"write_wal_micros_average\"})" ] }, { "title": "Compaction operations", "subtitle": "The count of compaction and flush operations", "expressions": [ "sum(rate(tikv_engine_event_total{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ] }, { "title": "WAL sync duration", "subtitle": "The time consumed when executing WAL sync operation", "expressions": [ "max(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_max\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile99\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_percentile95\"})", "avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\"$instance\", db=\"$db\",type=\"wal_file_sync_average\"})" ] }, { "title": "Compaction guard actions", "subtitle": "Compaction guard actions", "expressions": [ "sum(rate(tikv_raftstore_compaction_guard_action_total{instance=~\"$instance\", cf=~\"default|write\"}[1m])) by (cf, type)" ] }, { "title": "Compaction duration", "subtitle": "The time consumed when executing the compaction and flush operations", "expressions": [ "max(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_max\"})", "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile99\"})", "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_percentile95\"})", "avg(tikv_engine_compaction_time{instance=~\"$instance\", db=\"$db\",type=\"compaction_time_average\"})" ] }, { "title": "SST read duration", "subtitle": "The time consumed when reading SST files", "expressions": [ "max(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_max\"})", "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile99\"})", "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_percentile95\"})", "avg(tikv_engine_sst_read_micros{instance=~\"$instance\", db=\"$db\", type=\"sst_read_micros_average\"})" ] }, { "title": "Compaction reason", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_compaction_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (cf, reason)" ] }, { "title": "Block cache size", "subtitle": "The block cache size. Broken down by column family if shared block cache is disabled.", "expressions": [ "topk(20, avg(tikv_engine_block_cache_size_bytes{instance=~\"$instance\", db=\"$db\"}) by(cf, instance))" ] }, { "title": "Memtable hit", "subtitle": "The hit rate of memtable", "expressions": [ "sum(rate(tikv_engine_memtable_efficiency{instance=~\"$instance\", db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))" ] }, { "title": "Block cache flow", "subtitle": "The flow of different kinds of block cache operations", "expressions": [ "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))" ] }, { "title": "Block cache hit", "subtitle": "The hit rate of block cache", "expressions": [ "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_miss\"}[1m])))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", "sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{instance=~\"$instance\", db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))" ] }, { "title": "Keys flow", "subtitle": "The flow of different kinds of operations on keys", "expressions": [ "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_read\"}[1m]))", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"keys_written\"}[1m]))", "sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\"$instance\", db=\"$db\"}[1m]))" ] }, { "title": "Block cache operations", "subtitle": "The count of different kinds of block cache operations", "expressions": [ "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_data_add\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_index_add\"}[1m]))", "sum(rate(tikv_engine_cache_efficiency{instance=~\"$instance\", db=\"$db\", type=\"block_cache_add_failures\"}[1m]))" ] }, { "title": "Read flow", "subtitle": "The flow rate of read operations per type", "expressions": [ "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"iter_bytes_read\"}[1m]))" ] }, { "title": "Total keys", "subtitle": "The count of keys in each column family", "expressions": [ "sum(tikv_engine_estimate_num_keys{instance=~\"$instance\", db=\"$db\"}) by (cf)" ] }, { "title": "Write flow", "subtitle": "The flow of different kinds of write operations", "expressions": [ "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"wal_file_bytes\"}[1m]))", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))" ] }, { "title": "Bytes / Read", "subtitle": "The bytes per read", "expressions": [ "max(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_max\"})", "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile99\"})", "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_percentile95\"})", "avg(tikv_engine_bytes_per_read{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_read_average\"})" ] }, { "title": "Compaction flow", "subtitle": "The flow rate of compaction operations per type", "expressions": [ "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_read\"}[1m]))", "sum(rate(tikv_engine_compaction_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"bytes_written\"}[1m]))", "sum(rate(tikv_engine_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"flush_write_bytes\"}[1m]))" ] }, { "title": "Bytes / Write", "subtitle": "The bytes per write", "expressions": [ "max(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_max\"})", "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile99\"})", "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_percentile95\"})", "avg(tikv_engine_bytes_per_write{instance=~\"$instance\", db=\"$db\",type=\"bytes_per_write_average\"})" ] }, { "title": "Read amplication", "subtitle": "The read amplification per TiKV instance \t", "expressions": [ "sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\"$instance\", db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (instance)" ] }, { "title": "Compaction pending bytes", "subtitle": "The pending bytes to be compacted", "expressions": [ "sum(tikv_engine_pending_compaction_bytes{instance=~\"$instance\", db=\"$db\"}) by (cf)" ] }, { "title": "Number of snapshots", "subtitle": "The number of snapshot of each TiKV instance", "expressions": [ "tikv_engine_num_snapshots{instance=~\"$instance\", db=\"$db\"}" ] }, { "title": "Compression ratio", "subtitle": "The compression ratio of each level", "expressions": [ "avg(tikv_engine_compression_ratio{instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ] }, { "title": "Number files at each level", "subtitle": "The number of SST files for different column families in each level", "expressions": [ "avg(tikv_engine_num_files_at_level{instance=~\"$instance\", db=\"$db\"}) by (cf, level)" ] }, { "title": "Oldest snapshots duration", "subtitle": "The time that the oldest unreleased snapshot survivals", "expressions": [ "tikv_engine_oldest_snapshot_duration{instance=~\"$instance\", db=\"$db\"}" ] }, { "title": "Stall conditions changed of each CF", "subtitle": "Stall conditions changed of each column family", "expressions": [ "tikv_engine_stall_conditions_changed{instance=~\"$instance\", db=\"$db\"}" ] }, { "title": "Ingest SST duration seconds", "subtitle": "The time consumed when ingesting SST files", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le))", "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\"$instance\", db=\"$db\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\"$instance\", db=\"$db\"}[1m]))" ] }, { "title": "Write Stall Reason", "subtitle": null, "expressions": [ "sum(increase(tikv_engine_write_stall_reason{instance=~\"$instance\", db=\"$db\"}[1m])) by (type)" ] }, { "title": "Write stall duration", "subtitle": "The time which is caused by write stall", "expressions": [ "max(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_max\"})", "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile99\"})", "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_percentile95\"})", "avg(tikv_engine_write_stall{instance=~\"$instance\", db=\"$db\", type=\"write_stall_average\"})" ] }, { "title": "Ingestion picked level", "subtitle": "The level that the external file ingests into", "expressions": [ "sum(delta(tikv_engine_ingestion_picked_level_bucket{instance=~\"$instance\", db=\"$db\"}[1m])) by (le)" ] }, { "title": "Memtable size", "subtitle": "The memtable size of each column family", "expressions": [ "avg(tikv_engine_memory_bytes{instance=~\"$instance\", db=\"$db\", type=\"mem-tables\"}) by (cf)" ] } ] }, { "title": "Raft Engine", "panels": [ { "title": "Rfengine Persist Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_persist_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_persist_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_persist_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_persist_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Rotate Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_rotate_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_rotate_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_rotate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_rotate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine WAL Write Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_wal_write_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_wal_write_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99999999, sum(rate(raft_engine_wal_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Apply Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_apply_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_apply_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Truncate Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_truncate_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_truncate_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_truncate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_truncate_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Fetch Entries Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_fetch_entries_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_fetch_entries_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_fetch_entries_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_fetch_entries_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Compact WAL Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_compact_wal_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_compact_wal_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_compact_wal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_compact_wal_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Task Snapshot Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_take_snapshot_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_take_snapshot_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_take_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_take_snapshot_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Region Write Batch Size", "subtitle": "The bytes per write", "expressions": [ "sum(rate(raft_engine_region_write_batch_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_region_write_batch_size_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(raft_engine_region_write_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_region_write_batch_size_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Log File GC Size", "subtitle": "The bytes per write", "expressions": [ "sum(rate(raft_engine_log_file_gc_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_log_file_gc_size_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(raft_engine_log_file_gc_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_log_file_gc_size_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Backup Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_backup_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_backup_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_backup_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_backup_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Rfengine Backup Counter", "subtitle": "The I/O flow rate", "expressions": [ "sum(rate(raft_engine_backup_counter{instance=~\"$instance\"}[1m])) by (status)" ] }, { "title": "Rfengine DFS Worker Unhealthy", "subtitle": "The I/O flow rate", "expressions": [ "1 - raft_engine_dfs_worker_healthy" ] }, { "title": "Write Duration", "subtitle": "The time used in write operation", "expressions": [ "sum(rate(raft_engine_write_duration_seconds_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_write_duration_seconds_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.95, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Operation", "subtitle": "The count of operations per second", "expressions": [ "sum(rate(raft_engine_write_apply_duration_seconds_count{instance=~\"$instance\"}[1m]))", "sum(rate(raft_engine_read_entry_duration_seconds_count{instance=~\"$instance\"}[1m]))", "sum(rate(raft_engine_read_message_duration_seconds_count{instance=~\"$instance\"}[1m]))" ] }, { "title": "Write Duration Breakdown (99%)", "subtitle": "99% duration breakdown of write operation", "expressions": [ "histogram_quantile(0.99, sum(rate(raft_engine_write_preprocess_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_write_apply_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Flow", "subtitle": "The I/O flow rate", "expressions": [ "sum(rate(raft_engine_write_size_sum{instance=~\"$instance\"}[1m]))", "sum(rate(raft_engine_background_rewrite_bytes_sum{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "WAL Duration Breakdown (999%)", "subtitle": "999% duration breakdown of WAL write operation", "expressions": [ "histogram_quantile(0.999, sum(rate(raft_engine_write_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_sync_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_allocate_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_rotate_log_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Bytes / Written", "subtitle": "The bytes per write", "expressions": [ "sum(rate(raft_engine_write_size_sum{instance=~\"$instance\"}[1m])) / sum(rate(raft_engine_write_size_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(raft_engine_write_size_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.999, sum(rate(raft_engine_write_size_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Other Durations (99%)", "subtitle": "The 99% duration of operations other than write", "expressions": [ "histogram_quantile(0.99, sum(rate(raft_engine_read_entry_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_read_message_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(raft_engine_purge_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "File Count", "subtitle": "The average number of files", "expressions": [ "avg(raft_engine_log_file_count{instance=~\"$instance\"}) by (type)", "avg(raft_engine_swap_file_count{instance=~\"$instance\"})" ] }, { "title": "Entry Count", "subtitle": "The average number of log entries", "expressions": [ "avg(raft_engine_log_entry_count{instance=~\"$instance\"}) by (type)" ] } ] }, { "title": "Titan - $titan_db", "panels": [ { "title": "Blob file count", "subtitle": null, "expressions": [ "sum(tikv_engine_titandb_num_live_blob_file{instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_num_obsolete_blob_file{instance=~\"$instance\", db=\"$titan_db\"})" ] }, { "title": "Blob file size", "subtitle": null, "expressions": [ "sum(tikv_engine_titandb_live_blob_file_size{instance=~\"$instance\", db=\"$titan_db\"})", "sum(tikv_engine_titandb_obsolete_blob_file_size{instance=~\"$instance\", db=\"$titan_db\"})" ] }, { "title": "Live blob size", "subtitle": null, "expressions": [ "sum(tikv_engine_titandb_live_blob_size{instance=~\"$instance\", db=\"$titan_db\"})" ] }, { "title": "Blob cache size", "subtitle": "The blob cache size.", "expressions": [ "topk(20, avg(tikv_engine_blob_cache_size_bytes{instance=~\"$instance\", db=\"$titan_db\"}) by(cf, instance))" ] }, { "title": "Blob cache hit", "subtitle": "The hit rate of block cache", "expressions": [ "sum(rate(tikv_engine_blob_cache_efficiency{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_cache_hit\"}[1m])) / (sum(rate(tikv_engine_blob_cache_efficiency{db=\"$titan_db\", type=\"blob_cache_hit\"}[1m])) + sum(rate(tikv_engine_blob_cache_efficiency{db=\"$titan_db\", type=\"blob_cache_miss\"}[1m])))" ] }, { "title": "Iter touched blob file count", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_average\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile95\"})", "avg(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_percentile99\"})", "max(tikv_engine_blob_iter_touch_blob_file_count{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_iter_touch_blob_file_count_max\"})" ] }, { "title": "Blob file discardable ratio distribution", "subtitle": null, "expressions": [ "sum(tikv_engine_titandb_blob_file_discardable_ratio{instance=~\"$instance\", db=\"$titan_db\"}) by (ratio)" ] }, { "title": "Blob key size", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_average\"})", "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile95\"})", "avg(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_percentile99\"})", "max(tikv_engine_blob_key_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_key_size_max\"})" ] }, { "title": "Blob value size", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_average\"})", "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile95\"})", "avg(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_percentile99\"})", "max(tikv_engine_blob_value_size{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_value_size_max\"})" ] }, { "title": "Blob get operations", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_get\"}[2m]))" ] }, { "title": "Blob get duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_get_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ] }, { "title": "Blob iter operations", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_seek\"}[2m]))", "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_prev\"}[2m]))", "sum(rate(tikv_engine_blob_locate{instance=~\"$instance\", db=\"$titan_db\", type=\"number_blob_next\"}[2m]))" ] }, { "title": "Blob seek duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_seek_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ] }, { "title": "Blob next duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_next_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ] }, { "title": "Blob prev duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_average\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile95\"}) by (type)", "avg(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_percentile99\"}) by (type)", "max(tikv_engine_blob_prev_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=~\".*_max\"}) by (type)" ] }, { "title": "Blob keys flow", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[30s])) by (type)" ] }, { "title": "Blob bytes flow", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[2m])) by (type)" ] }, { "title": "Blob file read duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_average\"})", "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile99\"})", "avg(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_percentile95\"})", "max(tikv_engine_blob_file_read_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_read_micros_max\"})" ] }, { "title": "Blob file write duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_average\"})", "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile99\"})", "avg(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_percentile95\"})", "max(tikv_engine_blob_file_write_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_write_micros_max\"})" ] }, { "title": "Blob file sync operations", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_file_synced{instance=~\"$instance\", db=\"$titan_db\"}[2m]))" ] }, { "title": "Blob file sync duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_average\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile95\"})", "avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_percentile99\"})", "max(tikv_engine_blob_file_sync_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_file_sync_micros_max\"})" ] }, { "title": "Blob GC action", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_gc_action_count{instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ] }, { "title": "Blob GC duration", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_average\"})", "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile95\"})", "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_percentile99\"})", "avg(tikv_engine_blob_gc_micros_seconds{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_micros_max\"})" ] }, { "title": "Blob GC keys flow", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"keys.*\"}[30s])) by (type)" ] }, { "title": "Blob GC bytes flow", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\"$instance\", db=\"$titan_db\", type=~\"bytes.*\"}[30s])) by (type)" ] }, { "title": "Blob GC input file size", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_average\"})", "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile95\"})", "avg(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_percentile99\"})", "max(tikv_engine_blob_gc_input_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_input_file_max\"})" ] }, { "title": "Blob GC output file size", "subtitle": null, "expressions": [ "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_average\"})", "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile95\"})", "avg(tikv_engine_blob_gc_output_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_percentile99\"})", "max(tikv_engine_blob_gc_outputt_file{instance=~\"$instance\", db=\"$titan_db\", type=\"blob_gc_output_file_max\"})" ] }, { "title": "Blob GC file count", "subtitle": null, "expressions": [ "sum(rate(tikv_engine_blob_gc_file_count{instance=~\"$instance\", db=\"$titan_db\"}[2m])) by (type)" ] } ] }, { "title": "Pessimistic Locking", "panels": [ { "title": "Lock Manager Thread CPU", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"waiter_manager.*\"}[1m])) by (instance, name)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"deadlock_detect.*\"}[1m])) by (instance, name)" ] }, { "title": "Lock Manager Handled tasks", "subtitle": null, "expressions": [ "sum(rate(tikv_lock_manager_task_counter{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "Waiter lifetime duration", "subtitle": null, "expressions": [ "sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\"$instance\"}[1m])) by (le))", "histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Lock Waiting Queue", "subtitle": null, "expressions": [ "sum(max_over_time(tikv_lock_manager_wait_table_status{instance=~\"$instance\"}[15s])) by (type)", "sum(max_over_time(tikv_lock_wait_queue_entries_gauge_vec{instance=~\"$instance\"}[15s])) by (type)" ] }, { "title": "Deadlock detect duration", "subtitle": null, "expressions": [ "sum(rate(tikv_lock_manager_detect_duration_sum{instance=~\"$instance\"}[1m])) / sum(rate(tikv_lock_manager_detect_duration_count{instance=~\"$instance\"}[1m]))", "histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{instance=~\"$instance\"}[1m])) by (le))" ] }, { "title": "Detect error", "subtitle": null, "expressions": [ "sum(rate(tikv_lock_manager_error_counter{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "cloud request", "subtitle": null, "expressions": [ "sum(rate(tikv_cloud_request_duration_seconds_count{instance=~\"$instance\"}[1m])) by (cloud, req)" ] }, { "title": "Total pessimistic locks memory size", "subtitle": null, "expressions": [ "tikv_pessimistic_lock_memory_size{}" ] }, { "title": "Deadlock detector leader", "subtitle": null, "expressions": [ "sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{instance=~\"$instance\"}[15s])) by (instance)" ] }, { "title": "In-memory pessimistic locking result", "subtitle": null, "expressions": [ "sum(rate(tikv_in_memory_pessimistic_locking{instance=~\"$instance\"}[1m])) by (result)" ] } ] }, { "title": "Resolved-TS", "panels": [ { "title": "Resolved TS Worker CPU", "subtitle": " \tThe CPU utilization of resolved ts worker", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"resolved_ts.*\"}[1m])) by (instance)" ] }, { "title": "Advance ts Worker CPU", "subtitle": " \tThe CPU utilization of advance ts worker", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"advance_ts.*\"}[1m])) by (instance)" ] }, { "title": "Scan lock Worker CPU", "subtitle": " \tThe CPU utilization of scan lock worker", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"inc_scan.*\"}[1m])) by (instance)" ] }, { "title": "Max Resolved TS gap", "subtitle": "The gap between resolved tso and current time", "expressions": [ "sum(tikv_resolved_ts_min_resolved_ts_gap_millis{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Max Leader Resolved TS gap", "subtitle": "The gap between resolved tso of leaders and current time", "expressions": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_gap_millis{instance=~\"$instance\"}) by (instance)" ] }, { "title": "Min Resolved TS Region", "subtitle": "The region that has minimal resolved ts", "expressions": [ "sum(tikv_resolved_ts_min_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)" ] }, { "title": "Min Leader Resolved TS Region", "subtitle": "The region that its leader has minimal resolved ts.", "expressions": [ "sum(tikv_resolved_ts_min_leader_resolved_ts_region{tidb_cluster=~\"$tidb_cluster.*\", instance=~\"$instance\"}) by (instance)" ] }, { "title": "Check leader duration", "subtitle": "The time consumed when handle a check leader request", "expressions": [ "sum(delta(tikv_resolved_ts_check_leader_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Observe region status", "subtitle": "The status of resolved-ts observe regions", "expressions": [ "sum(tikv_resolved_ts_region_resolve_status{instance=~\"$instance\"}) by (type)" ] }, { "title": "99% CheckLeader request region count", "subtitle": "Bucketed histogram of region count in a check leader request", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Fail advance ts count", "subtitle": "The count of fail to advance resolved-ts", "expressions": [ "sum(delta(tikv_resolved_ts_fail_advance_count{instance=~\"$instance\"}[1m])) by (instance, reason)" ] }, { "title": "Lock heap size", "subtitle": "Total bytes in memory of resolved-ts observe regions's lock heap", "expressions": [ "avg(tikv_resolved_ts_lock_heap_bytes{instance=~\"$instance\"}) by (instance)" ] }, { "title": "99% CheckLeader request size", "subtitle": "Bucketed histogram of the check leader request size", "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_size_bytes_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", "histogram_quantile(0.99, sum(rate(tikv_check_leader_request_item_count_bucket{instance=~\"$instance\"}[1m])) by (le, instance))" ] }, { "title": "Pending command size", "subtitle": "Total bytes of pending commands in the channel", "expressions": [ "avg(tikv_resolved_ts_channel_penging_cmd_bytes_total{instance=~\"$instance\"}) by (instance)" ] } ] }, { "title": "Memory", "panels": [ { "title": "Allocator Stats", "subtitle": null, "expressions": [ "tikv_allocator_stats{instance=~\"$instance\"}" ] } ] }, { "title": "Backup & Import", "panels": [ { "title": "Backup CPU Utilization", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"b.*k.*w.*k.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_io\"}[1m])) by (instance)", "tikv_backup_softlimit{instance=~\"$instance\"}" ] }, { "title": "Backup Thread Count", "subtitle": null, "expressions": [ "sum(tikv_backup_thread_pool_size{instance=~\"$instance\"}) by(instance)" ] }, { "title": "Backup Errors", "subtitle": null, "expressions": [ "delta(tikv_backup_error_counter{instance=~\"$instance\"}[1m])" ] }, { "title": "Backup Write CF SST Size", "subtitle": null, "expressions": [ "max(rate(tikv_backup_range_size_bytes_bucket{instance=~\"$instance\", cf=\"write\"}[1m])) by (le)" ] }, { "title": "Backup Default CF SST Size", "subtitle": null, "expressions": [ "max(rate(tikv_backup_range_size_bytes_bucket{instance=~\"$instance\", cf=\"default\"}[1m])) by (le)" ] }, { "title": "Backup SST Generation Throughput", "subtitle": null, "expressions": [ "sum(rate(tikv_backup_range_size_bytes_sum{instance=~\"$instance\"}[1m]))", "rate(tikv_backup_range_size_bytes_sum{}[1m])" ] }, { "title": "Backup Scan SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)" ] }, { "title": "Backup Scan SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=\"scan\"}[1m])) by (le)" ] }, { "title": "Backup Save SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\", type=~\"save.*\"}[1m])) by (le)" ] }, { "title": "Backup SST Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))", "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type))", "sum(rate(tikv_backup_range_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{instance=~\"$instance\"}[1m])) by (type)" ] }, { "title": "External Storage Create Duration", "subtitle": null, "expressions": [ "max(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "External Storage Create Duration", "subtitle": null, "expressions": [ "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,type))", "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{instance=~\"$instance\"}[1m])) by (le,type))" ] }, { "title": "Checksum Request Duration", "subtitle": null, "expressions": [ "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[1m])) by (le,req))", "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\"$instance\", req=~\"analyze.*|checksum.*\"}[1m])) by (le,req))" ] }, { "title": "IO Utilization", "subtitle": null, "expressions": [ "rate(node_disk_io_time_seconds_total{}[1m])" ] }, { "title": "Import CPU Utilization", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0", "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)" ] }, { "title": "Import Thread Count", "subtitle": null, "expressions": [ "count(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)" ] }, { "title": "Import Errors", "subtitle": null, "expressions": [ "delta(tikv_import_error_counter{instance=~\"$instance\"}[1m])" ] }, { "title": "Import RPC Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, avg(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\"}[1m])) by (le, request))", "histogram_quantile(0.5, sum(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\"}[1m])) by (le, request, instance))" ] }, { "title": "Import RPC Ops", "subtitle": null, "expressions": [ "sum(rate(tikv_import_rpc_duration_count{instance=~\"$instance\", request!=\"switch_mode\"}[30s])) by (instance, request)", "sum(rate(tikv_import_rpc_duration_count{instance=~\"$instance\", request!=\"switch_mode\"}[30s])) by (request)" ] }, { "title": "Import Write/Download RPC Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\", request=~\"download|write\"}[1m])) by (le)" ] }, { "title": "Import Wait Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"queue\"}[1m])) by (le)" ] }, { "title": "Import Read SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"read\"}[1m])) by (le)" ] }, { "title": "Import Rewrite SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_download_duration_bucket{instance=~\"$instance\", type=\"rewrite\"}[1m])) by (le)" ] }, { "title": "Import Ingest RPC Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_rpc_duration_bucket{instance=~\"$instance\", request=~\"ingest\"}[1m])) by (le)" ] }, { "title": "Import Ingest SST Duration", "subtitle": null, "expressions": [ "max(rate(tikv_import_ingest_duration_bucket{instance=~\"$instance\", type=~\"ingest\"}[1m])) by (le)" ] }, { "title": "Import Ingest SST Bytes", "subtitle": null, "expressions": [ "max(rate(tikv_import_ingest_byte{instance=~\"$instance\"}[1m])) by (le)" ] }, { "title": "Import Download SST Throughput", "subtitle": null, "expressions": [ "sum(rate(tikv_import_download_bytes_sum{instance=~\"$instance\"}[1m])) by (instance)", "sum(rate(tikv_import_download_bytes_sum{instance=~\"$instance\"}[1m]))" ] }, { "title": "Import Local Write keys", "subtitle": null, "expressions": [ "delta(tikv_import_local_write_keys{instance=~\"$instance\"}[1m])" ] }, { "title": "Import Local Write bytes", "subtitle": null, "expressions": [ "delta(tikv_import_local_write_bytes{instance=~\"$instance\"}[1m])" ] }, { "title": "TTL Expired", "subtitle": "The accumulated TTL expired KV count during backup", "expressions": [ "sum(tikv_backup_raw_expired_count{instance=~\"$instance\"}) by (instance)", "sum(tikv_backup_raw_expired_count{instance=~\"$instance\"})" ] } ] }, { "title": "Encryption", "panels": [ { "title": "Encryption data keys", "subtitle": "Total number of encryption data keys in use", "expressions": [ "tikv_encryption_data_key_storage_total{}" ] }, { "title": "Encrypted files", "subtitle": "Number of files being encrypted", "expressions": [ "tikv_encryption_file_num{}" ] }, { "title": "Encryption initialized", "subtitle": "Flag to indicate if encryption is initialized", "expressions": [ "tikv_encryption_is_initialized{}" ] }, { "title": "Encryption meta files size", "subtitle": "Total size of encryption meta files", "expressions": [ "tikv_encryption_meta_file_size_bytes{}" ] }, { "title": "Encrypt/decrypt data nanos", "subtitle": null, "expressions": [ "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"encrypt_data_nanos\"}[1m])) by (req)", "sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\"$instance\" ,metric=\"decrypt_data_nanos\"}[1m])) by (req)" ] }, { "title": "Read/write encryption meta duration", "subtitle": "Writing or reading file duration (second)", "expressions": [ "histogram_quantile(1, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, operation))", "histogram_quantile(0.95, sum(rate(tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, type, operation))", "sum(rate(tikv_encryption_write_read_file_duration_seconds_sum{instance=~\"$instance\"}[1m])) by (le, type, operation) / sum(rate(tikv_encryption_write_read_file_duration_seconds_count{instance=~\"$instance\"}[1m])) by (le, type, operation)" ] } ] }, { "title": "Backup Log", "panels": [ { "title": "Endpoint Status", "subtitle": null, "expressions": [ "tikv_log_backup_enabled{instance=~\"$instance\"}" ] }, { "title": "Average Flush Size ", "subtitle": "The average flush size of last 30mins.", "expressions": [ "increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]) / on(instance) increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])" ] }, { "title": "Flushed Files (Last 30m) Per Host", "subtitle": "The current total flushed file number of this run.", "expressions": [ "round(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m]))" ] }, { "title": "Flush Times (Last 30m)", "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "round(sum(increase(tikv_log_backup_flush_duration_sec_count{stage=~\"save_files\",instance=~\"$instance\"}[30m])))" ] }, { "title": "Total Flushed Size (Last 30m)", "subtitle": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "sum(increase(tikv_log_backup_flush_file_size_sum{instance=~\"$instance\"}[30m]))" ] }, { "title": "Task Status", "subtitle": null, "expressions": [ "min(tikv_log_backup_task_status{instance=~\"$instance\"})" ] }, { "title": "Advancer Owner", "subtitle": null, "expressions": [ "tidb_log_backup_advancer_owner > 0" ] }, { "title": "Flush Files (Last 30m)", "subtitle": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot. \n**NOTE: The size may get reduced if some of TiKVs reboot.**", "expressions": [ "round(sum(increase(tikv_log_backup_flush_file_size_count{instance=~\"$instance\"}[30m])))" ] }, { "title": "CPU Usage", "subtitle": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{instance=~\"$instance\", name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}[2m])) by (instance)" ] }, { "title": "Handle Event Rate", "subtitle": null, "expressions": [ "rate(tikv_log_backup_handle_kv_batch_sum{instance=~\"$instance\"}[1m])" ] }, { "title": "Initial Scan Generate Event Throughput", "subtitle": "The data rate of initial scanning emitting events.", "expressions": [ "rate(tikv_log_backup_incremental_scan_bytes_sum{instance=~\"$instance\"}[$__rate_interval])" ] }, { "title": "Abnormal Checkpoint TS Lag", "subtitle": null, "expressions": [ "time() * 1000 - max(tidb_log_backup_last_checkpoint / 262144 > 0) by (task)", "time() * 1000" ] }, { "title": "Memory Of Events", "subtitle": "The estimated memory usage by the streaming backup module.", "expressions": [ "tikv_log_backup_heap_memory{instance=~\"$instance\"}" ] }, { "title": "Observed Region Count", "subtitle": null, "expressions": [ "tikv_log_backup_observed_region{instance=~\"$instance\"}", "sum(tikv_log_backup_observed_region{instance=~\"$instance\"})" ] }, { "title": "Errors", "subtitle": "The errors met when backing up.\n**They are retryable, don't worry.**", "expressions": [ "increase(tikv_log_backup_errors{instance=~\"$instance\"}[$__interval])", "tikv_log_backup_errors{instance=~\"$instance\"}" ] }, { "title": "Checkpoint TS of Tasks", "subtitle": null, "expressions": [ "max(tidb_log_backup_last_checkpoint{instance=~\"$instance\"} / 262144 > 0) by (task)", "time() * 1000" ] }, { "title": "Fatal Errors", "subtitle": "The errors met when backing up.", "expressions": [ "increase(tikv_log_backup_fatal_errors{instance=~\"$instance\"}[$__interval])" ] }, { "title": "Flush Duration", "subtitle": "The duration of flushing a batch of file.", "expressions": [ "sum(increase(tikv_log_backup_flush_duration_sec_bucket{instance=~\"$instance\", stage=~\"save_files\"}[$__interval])) by (le)" ] }, { "title": "Initial scanning duration", "subtitle": "The duration of scanning the initial data from local DB and transform them into apply events. \n", "expressions": [ "sum(increase(tikv_log_backup_initial_scan_duration_sec_bucket{instance=~\"$instance\"}[$__interval])) by (le)" ] }, { "title": "Convert Raft Event duration", "subtitle": "The duration of converting a raft request into a apply event. \n*This duration is for consuming a batch of events.*", "expressions": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"to_stream_event\"}[$__interval])) by (le)" ] }, { "title": "Wait for Lock Duration", "subtitle": "The duration of waiting the mutex of the controller. \n*This duration is for consuming a batch of events.*", "expressions": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"get_router_lock\"}[$__interval])) by (le)" ] }, { "title": "Command Batch Size", "subtitle": "The number of KV-modify of each raft command observed.", "expressions": [ "sum(increase(tikv_log_backup_handle_kv_batch_bucket{instance=~\"$instance\"}[$__interval])) by (le)" ] }, { "title": "Save to Temp File Duration", "subtitle": "The total cost of saving an event into temporary file. \n*This duration is for consuming a batch of events.*", "expressions": [ "sum(increase(tikv_log_backup_event_handle_duration_sec_bucket{instance=~\"$instance\", stage=~\"save_to_temp_file\"}[$__interval])) by (le)" ] }, { "title": "Write to Temp File Duration", "subtitle": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task. \n*This duration is for consuming a batch of events, for one region or one table.*", "expressions": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{instance=~\"$instance\", stage=\"write_to_tempfile\"}[$__interval])) by (le)" ] }, { "title": "System Write Call Duration", "subtitle": "The duration of collecting metadata and call the UNIX system call *write* for each event. \n*This duration is for consuming a batch of events, for one region or one table.*", "expressions": [ "sum(increase(tikv_log_backup_on_event_duration_seconds_bucket{instance=~\"$instance\", stage=\"syscall_write\"}[$__interval])) by (le)" ] }, { "title": "Internal Message Type", "subtitle": "The internal message type count.", "expressions": [ "sum(rate(tikv_log_backup_interal_actor_acting_duration_sec_count{instance=~\"$instance\"}[$__rate_interval])) by (message)" ] }, { "title": "Internal Message Handling Duration (P99)", "subtitle": "The internal handling message duration.", "expressions": [ "sum(histogram_quantile(0.99, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ] }, { "title": "Internal Message Handling Duration (P90)", "subtitle": "The internal handling message duration.", "expressions": [ "sum(histogram_quantile(0.9, rate(tikv_log_backup_interal_actor_acting_duration_sec_bucket{instance=~\"$instance\"}[10m]))) by (message)" ] }, { "title": "Initial Scan RocksDB Throughput ", "subtitle": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", "expressions": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op=~\"read_bytes\"}[$__rate_interval])) BY (op, cf)" ] }, { "title": "Initial Scan RocksDB Operation ", "subtitle": "Misc statistics of RocksDB during initial scanning.", "expressions": [ "sum(rate(tikv_log_backup_initial_scan_operations{instance=~\"$instance\", op!~\"read_bytes\"}[$__rate_interval])) BY (op, cf) > 0" ] }, { "title": "Initial Scanning Trigger Reason", "subtitle": "The reason of triggering initial scanning.", "expressions": [ "sum(increase(tikv_log_backup_initial_scan_reason{instance=~\"$instance\"}[$__rate_interval])) by (reason)" ] }, { "title": "Region Checkpoint Key Putting", "subtitle": null, "expressions": [ "sum(rate(tikv_log_backup_metadata_key_operation{instance=~\"$instance\"}[$__rate_interval])) by (type)" ] }, { "title": "Request Checkpoint Batch Size", "subtitle": null, "expressions": [ "sum(increase(tidb_log_backup_advancer_batch_size_bucket{type=\"checkpoint\"}[$__interval])) by (le)" ] }, { "title": "Tick Duration", "subtitle": null, "expressions": [ "sum(increase(tidb_log_backup_advancer_tick_duration_sec_bucket{step=~\"tick\"}[$__interval])) by (le)" ] }, { "title": "Region Checkpoint Failure Reason", "subtitle": "The reason of advancer failed to be advanced.", "expressions": [ "sum(increase(tidb_log_backup_region_request_failure{reason!=\"retryable-scan-region\"}[$__interval])) by (reason)" ] }, { "title": "Request Result", "subtitle": "The result of getting region checkpoints.", "expressions": [ "sum(increase(tidb_log_backup_region_request[$__interval])) by (result)" ] }, { "title": "Tick Duration (P99)", "subtitle": "The internal handling message duration.", "expressions": [ "sum(histogram_quantile(0.99, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ] }, { "title": "Tick Duration (P90)", "subtitle": "The internal handling message duration.", "expressions": [ "sum(histogram_quantile(0.9, rate(tidb_log_backup_advancer_tick_duration_sec_bucket[10m]))) by (step)" ] }, { "title": "Get Region Operation Count", "subtitle": "The frequent of getting region level checkpoint.", "expressions": [ "rate(tidb_log_backup_advancer_tick_duration_sec_count{step=\"get-regions-in-range\"}[$__rate_interval])" ] }, { "title": "Try Advance Trigger Time", "subtitle": "The variant of checkpoint group.", "expressions": [ "increase(tidb_log_backup_advancer_tick_duration_sec_count{step=\"try-advance\"}[$__interval])" ] } ] } ] }