// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. // Modifications Copyright 2024 PingCAP, Inc. use lazy_static::lazy_static; use prometheus::*; use prometheus_static_metric::*; make_auto_flush_static_metric! { pub label_enum RaftEntryFetchKind { async_fetch, sync_fetch, fallback_fetch, fetch_invalid, fetch_unused, } pub struct RaftEntryFetches : LocalIntCounter { "type" => RaftEntryFetchKind, } pub label_enum RaftEntryCryptType { encrypt, decrypt, } pub struct RaftEntryCryptDurationHistogram : LocalHistogram { "type" => RaftEntryCryptType, } } make_static_metric! { pub struct StoreBusyOnApplyRegionsGaugeVec: IntGauge { "type" => { busy_apply_peers, completed_apply_peers, }, } pub struct StoreBusyStateGaugeVec: IntGauge { "type" => { raftstore_busy, applystore_busy, }, } pub struct StoreCpuPoolGaugeVec: IntGauge { "pool" => { unified_read, scheduler, }, "source" => { store_level, region_sum, }, } pub struct RaftRouterDroppedCounterVec: IntCounter { "type" => { full, disconnected, }, } pub struct ColdRegionStateTransitionCounterVec: IntCounter { "state" => { regular, temp_regular, cold, }, } pub struct ColdCampaignEventCounterVec: IntCounter { "reason" => { read_index, leader_unknown, store_down, }, } } lazy_static! { pub static ref STORE_PROPOSE_SWITCH_MEM_TABLE_COUNTER: IntCounter = register_int_counter!( "rfstore_propose_switch_mem_table_counter", "Counter of rfstore propose switch mem table", ) .unwrap(); pub static ref IDLE_PEER_COUNT: IntGauge = register_int_gauge!("rfstore_idle_peers_count", "The number of idle peers").unwrap(); pub static ref COLD_REGION_COUNT: IntGauge = register_int_gauge!( "rfstore_cold_regions_count", "The number of local regions whose peer is in the Cold state." ) .unwrap(); pub static ref COLD_REGION_STATE_TRANSITION_COUNTER_VEC: ColdRegionStateTransitionCounterVec = register_static_int_counter_vec!( ColdRegionStateTransitionCounterVec, "rfstore_cold_region_state_transitions_total", "Total number of cold-region state transitions by destination state.", &["state"] ) .unwrap(); pub static ref COLD_LEADER_PROBE_PENDING_COUNT: IntGauge = register_int_gauge!( "rfstore_cold_leader_probe_pending_count", "The number of cold-region leader probes waiting for their timeout." ) .unwrap(); pub static ref COLD_CAMPAIGN_EVENT_COUNTER_VEC: ColdCampaignEventCounterVec = register_static_int_counter_vec!( ColdCampaignEventCounterVec, "rfstore_cold_campaign_events_total", "Total number of cold-region campaign attempts by reason.", &["reason"] ) .unwrap(); pub static ref LOCK_CACHE_LEN: IntGauge = register_int_gauge!("rfstore_lock_cache_len", "The length of lock cache").unwrap(); pub static ref LOCK_CACHE_CAPCITY: IntGauge = register_int_gauge!("rfstore_lock_cache_capacity", "The capacity of lock cache").unwrap(); pub static ref STARTUP_BATCH_LOADED_PEERS_GAUGE: IntGauge = register_int_gauge!( "rfstore_startup_batch_loaded_peers", "The number of local peers loaded during store startup." ) .unwrap(); pub static ref STARTUP_BATCH_ENABLED_GAUGE: IntGauge = register_int_gauge!( "rfstore_startup_batch_enabled", "Whether deterministic peer startup batching is enabled for the current store." ) .unwrap(); pub static ref STARTUP_BATCH_CURRENT_BATCH_GAUGE: IntGauge = register_int_gauge!( "rfstore_startup_batch_current_batch", "The currently active startup batch id. -1 means no batch is waiting." ) .unwrap(); pub static ref STARTUP_BATCH_CURRENT_BATCH_PEERS_GAUGE: IntGauge = register_int_gauge!( "rfstore_startup_batch_current_batch_peers", "The number of peers in the currently active startup batch." ) .unwrap(); pub static ref STARTUP_BATCH_CURRENT_BATCH_CONNECTED_PEERS_GAUGE: IntGauge = register_int_gauge!( "rfstore_startup_batch_current_batch_connected_peers", "The number of peers in the current startup batch that have observed raft connectivity." ) .unwrap(); pub static ref STARTUP_BATCH_TIMEOUT_COUNTER: IntCounter = register_int_counter!( "rfstore_startup_batch_timeout_total", "The number of startup batches advanced by timeout." ) .unwrap(); pub static ref STARTUP_BATCH_WAIT_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_startup_batch_wait_duration_seconds", "Bucketed histogram of startup batch wait durations before advancing to the next batch.", exponential_buckets(0.1, 2.0, 12).unwrap() ) .unwrap(); pub static ref STORE_INGEST_CONVERT_TASK_STATUS: GaugeVec = register_gauge_vec!( "rfstore_ingest_convert_sst_task_status_counter", "Counter of rfstore ingest convert sst task status", &["status"] ) .unwrap(); pub static ref STORE_RECV_MSGS_COUNT_HISTOGRAM: Histogram = register_histogram!( "rfstore_receive_msgs_count", "Bucketed histogram of receive messages count", exponential_buckets(1.0, 2.0, 20).unwrap() ) .unwrap(); pub static ref STORE_RECV_MSGS_SIZE_HISTOGRAM: Histogram = register_histogram!( "rfstore_receive_msgs_size_bytes", "Bucketed histogram of receive messages size", exponential_buckets(512.0, 2.0, 20).unwrap() ) .unwrap(); pub static ref STORE_STORE_MSG_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_store_msg_duration_seconds", "Bucketed histogram of store message handling duration", exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref STORE_RECV_MSGS_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_receive_msgs_duration_seconds", "Bucketed histogram of receive messages duration", exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref STORE_SYNC_AUX_WORKER_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_sync_aux_worker_duration_seconds", "Bucketed histogram of syncing aux workers", exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref STORE_SEND_AUX_TASK_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_send_aux_task_duration_seconds", "Bucketed histogram of send aux task duration", exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref STORE_PROC_MSGS_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_process_msgs_duration_seconds", "Bucketed histogram of process messages duration", exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); // Raft entry fetches categorized by kind: async, sync, fallback, invalid, unused pub static ref RAFT_ENTRY_FETCHES_VEC: IntCounterVec = register_int_counter_vec!( "rfstore_raft_entry_fetches_total", "Number of raft entry fetches by type", &["type"] ) .unwrap(); pub static ref RAFT_ENTRY_FETCHES: RaftEntryFetches = auto_flush_from!(RAFT_ENTRY_FETCHES_VEC, RaftEntryFetches); pub static ref RAFT_ENTRY_FETCHES_TASK_DURATION_HISTOGRAM: Histogram = register_histogram!( "rfstore_raft_entry_fetches_task_duration_seconds", "Bucketed histogram of raft entry fetches task duration.", exponential_buckets(0.0005, 2.0, 21).unwrap() // 500us ~ 8.7m ).unwrap(); pub static ref RAFT_ENTRY_CRYPT_DURATION_VEC: HistogramVec = register_histogram_vec!( "rfstore_raft_entry_crypt_duration_seconds", "Bucketed histogram of raft entry encryption/decryption duration.", &["type"], exponential_buckets(0.00001, 2.0, 26).unwrap() ).unwrap(); pub static ref RAFT_ENTRY_CRYPT_DURATION: RaftEntryCryptDurationHistogram = auto_flush_from!(RAFT_ENTRY_CRYPT_DURATION_VEC, RaftEntryCryptDurationHistogram); pub static ref STORE_BUSY_ON_APPLY_REGIONS_GAUGE_VEC: StoreBusyOnApplyRegionsGaugeVec = register_static_int_gauge_vec!( StoreBusyOnApplyRegionsGaugeVec, "tikv_raftstore_busy_on_apply_region_total", "Total number of regions busy on apply or complete apply.", &["type"] ).unwrap(); pub static ref STORE_PROCESS_BUSY_GAUGE_VEC: StoreBusyStateGaugeVec = register_static_int_gauge_vec!( StoreBusyStateGaugeVec, "tikv_raftstore_process_busy", "Is raft process busy or not", &["type"] ).unwrap(); pub static ref STORE_CPU_POOL_GAUGE_VEC: StoreCpuPoolGaugeVec = register_static_int_gauge_vec!( StoreCpuPoolGaugeVec, "tikv_rfstore_cpu_pool_usage", "CPU usage comparison between store-level thread stats and region-level resource metering per thread pool.", &["pool", "source"] ).unwrap(); pub static ref STORE_RAFT_APPLY_AHEAD_PERSIST_HISTOGRAM: Histogram = register_histogram!( "tikv_raft_apply_ahead_of_persist", "Histogram of the raft log lag between persisted index and applied index", exponential_buckets(1.0, 2.0, 20).unwrap() ).unwrap(); pub static ref STORE_RAFT_ENABLE_UNPERSISTED_APPLY_GAUGE: IntGauge = register_int_gauge!( "tikv_raft_enable_unpersisted_apply_regions", "The number of regions that enable apply unpersisted raft log." ).unwrap(); pub static ref ENCRYPTION_APPLY_PAUSED_PEERS: IntGauge = register_int_gauge!( "tikv_rfstore_encryption_apply_paused_peers", "Current number of peers paused because an encrypted raft entry cannot be decrypted." ).unwrap(); pub static ref APPLY_HANDLE_BATCH_COUNT_HISTOGRAM: Histogram = register_histogram!( "tikv_rfstore_apply_handle_msgs_batch_number", "The number of apply batch handled in one apply futher.", exponential_buckets(1.0, 2.0, 10).unwrap() ).unwrap(); pub static ref APPLY_POLL_DURATION_HISTOGRAM: Histogram = register_histogram!( "tikv_rfstore_apply_running_duration_secs", "Bucketed histogram of apply future running time duration.", exponential_buckets(0.00001, 2.0, 26).unwrap() ).unwrap(); pub static ref INGEST_TRIGGER_COMPACTION_COUNTER: IntCounter = register_int_counter!( "tikv_ingest_trigger_compaction_total", "Counter for how many times ingest triggered a new major compaction due to L0 overlap" ) .unwrap(); pub static ref INGEST_COMPACTION_PROPOSAL_FAILED_COUNTER: IntCounter = register_int_counter!( "tikv_ingest_compaction_proposal_failed_total", "Counter for how many times manual major compaction proposal failed for ingest" ) .unwrap(); pub static ref RAFT_TXN_FILE_COLLECT_PESSIMISTIC_LOCK_DUR: Histogram = register_histogram!( "tikv_rfstore_txn_file_collect_pessimistic_lock_dur", "Bucketed histogram of inspect duration.", exponential_buckets(0.001, 2.0, 15).unwrap() // 1ms ~ 32s. ) .unwrap(); pub static ref TXN_FILE_ACTIVE_RESOLVE_SCHEDULE_TOTAL: IntCounterVec = register_int_counter_vec!( "tikv_rfstore_txn_file_active_resolve_schedule_total", "Total number of active txn-file lock resolve scheduling outcomes", &["result"] ) .unwrap(); pub static ref TXN_FILE_ACTIVE_RESOLVE_TASK_DURATION_HISTOGRAM: Histogram = register_histogram!( "tikv_rfstore_txn_file_active_resolve_task_duration_seconds", "Bucketed histogram of active txn-file lock resolve task duration", exponential_buckets(0.001, 2.0, 20).unwrap() ) .unwrap(); pub static ref TXN_FILE_ACTIVE_RESOLVE_RUNNING_TASKS: IntGauge = register_int_gauge!( "tikv_rfstore_txn_file_active_resolve_running_tasks", "Current number of active txn-file lock resolve tasks in progress" ) .unwrap(); pub static ref TXN_FILE_ACTIVE_RESOLVE_CONCURRENCY_LIMIT: IntGauge = register_int_gauge!( "tikv_rfstore_txn_file_active_resolve_concurrency_limit", "Current configured concurrency limit for active txn-file lock resolve" ) .unwrap(); pub static ref TXN_FILE_ACTIVE_RESOLVE_LOCKS_TOTAL: IntCounterVec = register_int_counter_vec!( "tikv_rfstore_txn_file_active_resolve_locks_total", "Total number of active txn-file locks by resolve result", &["result"] ) .unwrap(); pub static ref TXN_FILE_SPLIT_DENIED_REGIONS_TOTAL: IntCounterVec = register_int_counter_vec!( "tikv_rfstore_txn_file_split_denied_regions_total", "Total number of regions where split was denied by txn-file locks", &["phase"] ) .unwrap(); pub static ref TXN_FILE_SPLIT_DENIED_LOCKS_TOTAL: IntCounterVec = register_int_counter_vec!( "tikv_rfstore_txn_file_split_denied_locks_total", "Total number of txn-file locks observed when split was denied", &["phase"] ) .unwrap(); pub static ref CHANNEL_FULL_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_channel_full_total", "Total number of channel full errors.", &["type"] ) .unwrap(); pub static ref RAFT_ROUTER_RAFT_MESSAGE_DROPPED_TOTAL: RaftRouterDroppedCounterVec = register_static_int_counter_vec!( RaftRouterDroppedCounterVec, "tikv_rfstore_router_raft_message_dropped_total", "Total raft messages dropped by the bounded raft router peer channel", &["type"] ).unwrap(); pub static ref DISK_PROBE_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_rfstore_disk_probe_duration_seconds", "Bucketed histogram of rfstore fail-fast disk probe duration.", &["disk", "outcome"], exponential_buckets(0.00001, 2.0, 26).unwrap() ).unwrap(); } #[inline] pub(crate) fn record_txn_file_split_denied(phase: &'static str, lock_count: usize) { TXN_FILE_SPLIT_DENIED_REGIONS_TOTAL .with_label_values(&[phase]) .inc(); TXN_FILE_SPLIT_DENIED_LOCKS_TOTAL .with_label_values(&[phase]) .inc_by(lock_count as u64); }