// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. // Modifications Copyright 2022 PingCAP, Inc. //! Prometheus metrics for storage functionality. use std::{cell::RefCell, mem, sync::Arc, time::Duration}; use collections::HashMap; use kvproto::{kvrpcpb::KeyRange, pdpb::QueryKind}; use lazy_static::lazy_static; use pd_client::BucketMeta; use prometheus::*; use prometheus_static_metric::*; use raftstore::store::{ReadStats, util::build_key_range}; use txn_types::Key; use crate::{ server::metrics::{GcKeysCF as ServerGcKeysCF, GcKeysDetail as ServerGcKeysDetail}, storage::kv::{FlowStatsReporter, Statistics}, }; struct StorageLocalMetrics { local_scan_details: HashMap, local_read_stats: ReadStats, read_stats_window_start: tikv_util::time::Instant, } thread_local! { static TLS_STORAGE_METRICS: RefCell = RefCell::new( StorageLocalMetrics { local_scan_details: HashMap::default(), local_read_stats:ReadStats::default(), read_stats_window_start: tikv_util::time::Instant::now_coarse(), } ); } /// Defers a query observation until its flow statistics are available. /// Dropping the guard still records the attempted query. pub(crate) struct ReadQueryGuard<'a> { region_id: u64, key_ranges: Option>, kind: QueryKind, } enum QueryRanges<'a> { One(KeyRange), Batch(&'a [Key]), } impl<'a> ReadQueryGuard<'a> { pub(crate) fn new(region_id: u64, key_range: KeyRange, kind: QueryKind) -> Self { Self { region_id, key_ranges: Some(QueryRanges::One(key_range)), kind, } } pub(crate) fn new_batch(region_id: u64, keys: &'a [Key], kind: QueryKind) -> Self { Self { region_id, key_ranges: Some(QueryRanges::Batch(keys)), kind, } } pub(crate) fn record(mut self) { self.record_inner(); } fn record_inner(&mut self) { let Some(key_ranges) = self.key_ranges.take() else { return; }; match key_ranges { QueryRanges::One(key_range) => { tls_collect_query_range(self.region_id, key_range, self.kind) } QueryRanges::Batch(keys) => { let key_ranges = keys .iter() .map(|key| build_key_range(key.as_encoded(), key.as_encoded(), false)) .collect(); tls_collect_query_batch(self.region_id, key_ranges, self.kind) } } } } impl Drop for ReadQueryGuard<'_> { fn drop(&mut self) { self.record_inner(); } } pub fn tls_flush(reporter: &R) { TLS_STORAGE_METRICS.with(|m| { let mut m = m.borrow_mut(); let now = tikv_util::time::Instant::now_coarse(); let report_interval = now.saturating_duration_since(m.read_stats_window_start); m.read_stats_window_start = now; for (cmd, stat) in m.local_scan_details.drain() { for (cf, cf_details) in stat.details_enum().iter() { for (tag, count) in cf_details.iter() { KV_COMMAND_SCAN_DETAILS_STATIC .get(cmd) .get((*cf).into()) .get((*tag).into()) .inc_by(*count as u64); } } } // Report PD metrics if !m.local_read_stats.is_empty() { let mut read_stats = mem::take(&mut m.local_read_stats); read_stats.set_report_interval(report_interval); reporter.report_read_stats(read_stats); } }); } // Read-pool metrics and load-based split are both evaluated once per second. // Sync point reads execute on caller threads without a metrics ticker, so // report their complete query and flow statistics at the same cadence. const SYNC_READ_STATS_REPORT_INTERVAL: Duration = Duration::from_secs(1); pub(crate) fn tls_maybe_take_sync_read_stats() -> Option { TLS_STORAGE_METRICS.with(|m| { let mut m = m.borrow_mut(); let now = tikv_util::time::Instant::now_coarse(); if m.local_read_stats.is_empty() { m.read_stats_window_start = now; return None; } let report_interval = now.saturating_duration_since(m.read_stats_window_start); if report_interval < SYNC_READ_STATS_REPORT_INTERVAL { return None; } m.read_stats_window_start = now; let mut read_stats = mem::take(&mut m.local_read_stats); // Preserve the previous sync-read behavior: bucket aggregation is // intentionally left to ticker-driven read-pool observations because // merging per-gRPC-thread bucket samples is comparatively expensive. read_stats.region_buckets.clear(); read_stats.set_report_interval(report_interval); Some(read_stats) }) } pub fn tls_collect_scan_details(cmd: CommandKind, stats: &Statistics) { TLS_STORAGE_METRICS.with(|m| { m.borrow_mut() .local_scan_details .entry(cmd) .or_default() .add(stats); }); } pub fn tls_collect_read_flow( region_id: u64, start: Option<&[u8]>, end: Option<&[u8]>, statistics: &Statistics, buckets: Option<&Arc>, ) { TLS_STORAGE_METRICS.with(|m| { let mut m = m.borrow_mut(); m.local_read_stats.add_flow( region_id, buckets, start, end, &statistics.write.flow_stats, &statistics.data.flow_stats, ); }); } pub fn tls_collect_query( region_id: u64, start_key: &[u8], end_key: &[u8], reverse_scan: bool, kind: QueryKind, ) { tls_collect_query_range( region_id, build_key_range(start_key, end_key, reverse_scan), kind, ); } fn tls_collect_query_range(region_id: u64, key_range: KeyRange, kind: QueryKind) { TLS_STORAGE_METRICS.with(|m| { m.borrow_mut() .local_read_stats .add_query_num(region_id, key_range, kind); }); } pub fn tls_collect_query_batch(region_id: u64, key_ranges: Vec, kind: QueryKind) { TLS_STORAGE_METRICS.with(|m| { let mut m = m.borrow_mut(); m.local_read_stats .add_query_num_batch(region_id, key_ranges, kind); }); } /// Adds query counts to the current TLS reporting window without sampling /// ranges. pub fn tls_collect_query_count(region_id: u64, kind: QueryKind, query_num: u64) { TLS_STORAGE_METRICS.with(|m| { m.borrow_mut() .local_read_stats .add_query_count(region_id, kind, query_num); }); } #[cfg(test)] mod tests { use super::*; fn reset_tls_storage_metrics() { TLS_STORAGE_METRICS.with(|m| { *m.borrow_mut() = StorageLocalMetrics { local_scan_details: HashMap::default(), local_read_stats: ReadStats::default(), read_stats_window_start: tikv_util::time::Instant::now_coarse(), }; }); } #[test] fn test_take_sync_read_stats_preserves_complete_observation() { reset_tls_storage_metrics(); for _ in 0..4 { tls_collect_query(1, b"a", b"z", false, QueryKind::Get); } let mut statistics = Statistics::default(); statistics.write.flow_stats.read_keys = 3; statistics.write.flow_stats.read_bytes = 128; tls_collect_read_flow(1, Some(b"a"), Some(b"z"), &statistics, None); TLS_STORAGE_METRICS.with(|m| { m.borrow_mut().read_stats_window_start = tikv_util::time::Instant::now_coarse() - Duration::from_secs(2); }); let read_stats = tls_maybe_take_sync_read_stats().unwrap(); let region_info = &read_stats.region_infos[&1]; assert_eq!(region_info.query_stats.get_read_query_num(), 4); assert!(region_info.get_read_qps() <= 2); assert!(!region_info.key_ranges.is_empty()); assert!( region_info .key_ranges .iter() .all(|range| range == &build_key_range(b"a", b"z", false)) ); assert_eq!(region_info.flow.read_keys, 3); assert_eq!(region_info.flow.read_bytes, 128); assert!(tls_maybe_take_sync_read_stats().is_none()); reset_tls_storage_metrics(); } #[test] fn test_read_query_guard_records_once_on_record_or_drop() { reset_tls_storage_metrics(); ReadQueryGuard::new(1, build_key_range(b"a", b"b", false), QueryKind::Get).record(); drop(ReadQueryGuard::new( 1, build_key_range(b"b", b"c", false), QueryKind::Get, )); let batch_keys = vec![ Key::from_encoded(b"c".to_vec()), Key::from_encoded(b"d".to_vec()), ]; ReadQueryGuard::new_batch(1, &batch_keys, QueryKind::Get).record(); let dropped_batch_keys = vec![Key::from_encoded(b"e".to_vec())]; drop(ReadQueryGuard::new_batch( 1, &dropped_batch_keys, QueryKind::Get, )); TLS_STORAGE_METRICS.with(|m| { let metrics = m.borrow(); let region_info = &metrics.local_read_stats.region_infos[&1]; assert_eq!(region_info.query_stats.get_read_query_num(), 5); assert_eq!(region_info.key_ranges.len(), 5); }); reset_tls_storage_metrics(); } } make_auto_flush_static_metric! { pub label_enum CommandKind { get, raw_batch_get_command, scan, batch_get, batch_get_command, prewrite, acquire_pessimistic_lock, acquire_pessimistic_lock_resumed, commit, cleanup, rollback, pessimistic_rollback, pessimistic_rollback_read_phase, txn_heart_beat, check_txn_status, check_secondary_locks, scan_lock, resolve_lock, resolve_lock_async, resolve_lock_background, resolve_lock_lite, delete_range, pause, key_mvcc, start_ts_mvcc, flashback_to_version, raw_get, raw_batch_get, raw_scan, raw_batch_scan, raw_put, raw_batch_put, raw_delete, raw_delete_range, raw_batch_delete, raw_get_key_ttl, raw_compare_and_swap, raw_atomic_store, raw_checksum, txn_file, get_lock_wait_info, unknown, } pub label_enum CommandStageKind { new, snapshot, async_snapshot_err, precheck_write_ok, precheck_write_err, snapshot_ok, snapshot_err, read_finish, next_cmd, lock_wait, process, prepare_write_err, write, write_finish, async_write_err, error, pipelined_write, pipelined_write_finish, async_apply_prewrite, async_apply_prewrite_finish, wakeup_deadline_exceeded, } pub label_enum CommandPriority { low, normal, high, } pub label_enum GcKeysCF { default, lock, write, } pub label_enum GcKeysDetail { processed_keys, get, next, prev, seek, seek_for_prev, over_seek_bound, next_tombstone, prev_tombstone, seek_tombstone, seek_for_prev_tombstone, raw_value_tombstone, } pub label_enum CheckMemLockResult { locked, unlocked, } pub label_enum InMemoryPessimisticLockingResult { success, full, } pub label_enum AsyncResolveLockDispatchResult { ok, err_full, err_closed, } pub label_enum AsyncResolveLockDequeueType { reason_txn_count, reason_min_wait, reason_inflight_finished, reason_max_wait, } pub label_enum AsyncResolveLockBgScheduleType { task, task_failed, txn, task_dropped, inflight_expired, inflight_missing, } pub label_enum AsyncResolveLockInqueueType { dedup, merged, inserted, } pub struct CommandScanDetails: LocalIntCounter { "req" => CommandKind, "cf" => GcKeysCF, "tag" => GcKeysDetail, } pub struct SchedDurationVec: LocalHistogram { "type" => CommandKind, } pub struct ProcessingReadVec: LocalHistogram { "type" => CommandKind, } pub struct KReadVec: LocalHistogram { "type" => CommandKind, } pub struct KvCommandCounterVec: LocalIntCounter { "type" => CommandKind, } pub struct SchedStageCounterVec: LocalIntCounter { "type" => CommandKind, "stage" => CommandStageKind, } pub struct SchedLatchDurationVec: LocalHistogram { "type" => CommandKind, } pub struct KvCommandKeysWrittenVec: LocalHistogram { "type" => CommandKind, } pub struct SchedTooBusyVec: LocalIntCounter { "type" => CommandKind, } pub label_enum SchedTooBusyReason { global, keyspace, } pub struct SchedTooBusyReasonVec: LocalIntCounter { "reason" => SchedTooBusyReason, } pub struct SchedCommandPriCounterVec: LocalIntCounter { "priority" => CommandPriority, } pub struct CheckMemLockHistogramVec: LocalHistogram { "type" => CommandKind, "result" => CheckMemLockResult, } pub struct TxnCommandThrottleTimeCounterVec: LocalIntCounter { "type" => CommandKind, } pub struct InMemoryPessimisticLockingCounter: LocalIntCounter { "result" => InMemoryPessimisticLockingResult, } pub struct AsyncResolveLockDispatchCounterVec: LocalIntCounter { "result" => AsyncResolveLockDispatchResult, } pub struct AsyncResolveLockDequeueCounterVec: LocalIntCounter { "type" => AsyncResolveLockDequeueType, } pub struct AsyncResolveLockBgScheduleCounterVec: LocalIntCounter { "type" => AsyncResolveLockBgScheduleType, } pub struct AsyncResolveLockInqueueCounterVec: LocalIntCounter { "type" => AsyncResolveLockInqueueType, } pub label_enum TxnFileStage { prepare, } pub struct SchedTxnFileDurationVec: LocalHistogram { "type" => CommandKind, "stage" => TxnFileStage, } } impl From for GcKeysCF { fn from(cf: ServerGcKeysCF) -> GcKeysCF { match cf { ServerGcKeysCF::default => GcKeysCF::default, ServerGcKeysCF::lock => GcKeysCF::lock, ServerGcKeysCF::write => GcKeysCF::write, } } } impl From for GcKeysDetail { fn from(detail: ServerGcKeysDetail) -> GcKeysDetail { match detail { ServerGcKeysDetail::processed_keys => GcKeysDetail::processed_keys, ServerGcKeysDetail::get => GcKeysDetail::get, ServerGcKeysDetail::next => GcKeysDetail::next, ServerGcKeysDetail::prev => GcKeysDetail::prev, ServerGcKeysDetail::seek => GcKeysDetail::seek, ServerGcKeysDetail::seek_for_prev => GcKeysDetail::seek_for_prev, ServerGcKeysDetail::over_seek_bound => GcKeysDetail::over_seek_bound, ServerGcKeysDetail::next_tombstone => GcKeysDetail::next_tombstone, ServerGcKeysDetail::prev_tombstone => GcKeysDetail::prev_tombstone, ServerGcKeysDetail::seek_tombstone => GcKeysDetail::seek_tombstone, ServerGcKeysDetail::seek_for_prev_tombstone => GcKeysDetail::seek_for_prev_tombstone, ServerGcKeysDetail::raw_value_tombstone => GcKeysDetail::raw_value_tombstone, } } } make_static_metric! { pub struct LockUpgradeConflictCounter: IntCounter { "reason" => { duplicate_in_flight, second_upgrader, }, } pub struct LockWaitQueueEntriesGauge: IntGauge { "type" => { waiters, keys, }, } pub struct TxnStatusCacheSizeGauge: IntGauge { "type" => { used, allocated, } } pub struct AsyncResolveLockQueueSizeGauge: IntGauge { "type" => { pending_batch, pending_txn, inflight_batch, blocked_batch, }, } } lazy_static! { pub static ref KV_COMMAND_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_storage_command_total", "Total number of commands received.", &["type"] ) .unwrap(); pub static ref KV_COMMAND_COUNTER_VEC_STATIC: KvCommandCounterVec = auto_flush_from!(KV_COMMAND_COUNTER_VEC, KvCommandCounterVec); pub static ref SCHED_STAGE_COUNTER: IntCounterVec = { register_int_counter_vec!( "tikv_scheduler_stage_total", "Total number of commands on each stage.", &["type", "stage"] ) .unwrap() }; pub static ref SCHED_STAGE_COUNTER_VEC: SchedStageCounterVec = auto_flush_from!(SCHED_STAGE_COUNTER, SchedStageCounterVec); pub static ref ASYNC_RESOLVE_LOCK_DISPATCH_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_async_resolve_lock_dispatch_total", "Total number of async resolve lock task dispatch attempts by result.", &["result"] ) .unwrap(); pub static ref ASYNC_RESOLVE_LOCK_DISPATCH_COUNTER_VEC_STATIC: AsyncResolveLockDispatchCounterVec = auto_flush_from!( ASYNC_RESOLVE_LOCK_DISPATCH_COUNTER_VEC, AsyncResolveLockDispatchCounterVec ); pub static ref ASYNC_RESOLVE_LOCK_DEQUEUE_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_async_resolve_lock_dequeue_total", "Total number of async resolve lock batches dequeued from pending queue by type.", &["type"] ) .unwrap(); pub static ref ASYNC_RESOLVE_LOCK_DEQUEUE_COUNTER_VEC_STATIC: AsyncResolveLockDequeueCounterVec = auto_flush_from!( ASYNC_RESOLVE_LOCK_DEQUEUE_COUNTER_VEC, AsyncResolveLockDequeueCounterVec ); pub static ref ASYNC_RESOLVE_LOCK_BG_SCHEDULE_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_async_resolve_lock_bg_schedule_total", "Total number of async resolve lock background scheduled tasks, scheduled txn statuses, dropped tasks, and inflight cleanup events by type.", &["type"] ) .unwrap(); pub static ref ASYNC_RESOLVE_LOCK_BG_SCHEDULE_COUNTER_VEC_STATIC: AsyncResolveLockBgScheduleCounterVec = auto_flush_from!( ASYNC_RESOLVE_LOCK_BG_SCHEDULE_COUNTER_VEC, AsyncResolveLockBgScheduleCounterVec ); pub static ref ASYNC_RESOLVE_LOCK_INQUEUE_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_async_resolve_lock_inqueue_total", "Total number of async resolve lock task inqueue results.", &["type"] ) .unwrap(); pub static ref ASYNC_RESOLVE_LOCK_INQUEUE_COUNTER_VEC_STATIC: AsyncResolveLockInqueueCounterVec = auto_flush_from!( ASYNC_RESOLVE_LOCK_INQUEUE_COUNTER_VEC, AsyncResolveLockInqueueCounterVec ); pub static ref ASYNC_RESOLVE_LOCK_QUEUE_SIZE_GAUGE_VEC: AsyncResolveLockQueueSizeGauge = register_static_int_gauge_vec!( AsyncResolveLockQueueSizeGauge, "tikv_scheduler_async_resolve_lock_queue_size", "Current async resolve lock queue size by type.", &["type"] ) .unwrap(); pub static ref SCHED_WRITING_BYTES_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_writing_bytes", "Total number of writing kv." ) .unwrap(); pub static ref SCHED_CONTEXT_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_context_total", "Total number of pending commands." ) .unwrap(); pub static ref SCHED_WRITE_FLOW_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_write_flow", "The write flow passed through at scheduler level." ) .unwrap(); pub static ref SCHED_THROTTLE_FLOW_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_throttle_flow", "The throttled write flow at scheduler level." ) .unwrap(); pub static ref SCHED_L0_TARGET_FLOW_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_l0_target_flow", "The target flow of L0." ) .unwrap(); pub static ref SCHED_MEMTABLE_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_memtable", "The number of memtables.", &["cf"] ) .unwrap(); pub static ref SCHED_L0_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_l0", "The number of l0 files.", &["cf"] ) .unwrap(); pub static ref SCHED_L0_AVG_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_l0_avg", "The number of average l0 files.", &["cf"] ) .unwrap(); pub static ref SCHED_FLUSH_FLOW_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_flush_flow", "The speed of flush flow.", &["cf"] ) .unwrap(); pub static ref SCHED_L0_FLOW_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_l0_flow", "The speed of l0 compaction flow.", &["cf"] ) .unwrap(); pub static ref SCHED_THROTTLE_ACTION_COUNTER: IntCounterVec = { register_int_counter_vec!( "tikv_scheduler_throttle_action_total", "Total number of actions for flow control.", &["cf", "type"] ) .unwrap() }; pub static ref SCHED_DISCARD_RATIO_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_discard_ratio", "The discard ratio for flow control." ) .unwrap(); pub static ref SCHED_THROTTLE_CF_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_throttle_cf", "The CF being throttled.", &["cf"] ).unwrap(); pub static ref SCHED_PENDING_COMPACTION_BYTES_GAUGE: IntGaugeVec = register_int_gauge_vec!( "tikv_scheduler_pending_compaction_bytes", "The number of pending compaction bytes.", &["type"] ) .unwrap(); pub static ref SCHED_TOKIO_POOL_MEAN_FIRST_POLL_DELAY: Gauge = register_gauge!( "tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds", "Mean time from task spawn to first poll (schedule wait time) for scheduler tokio pool" ) .unwrap(); pub static ref SCHED_TOKIO_POOL_FIRST_POLL_COUNT: IntCounter = register_int_counter!( "tikv_scheduler_tokio_pool_first_poll_total", "Total number of tasks polled for first time in scheduler tokio pool" ) .unwrap(); pub static ref SCHED_TOKIO_POOL_MEAN_IDLE_DURATION: Gauge = register_gauge!( "tikv_scheduler_tokio_pool_mean_idle_duration_seconds", "Mean time tasks spent idle waiting on external events (I/O) for scheduler tokio pool" ) .unwrap(); pub static ref SCHED_TOKIO_POOL_MEAN_SCHEDULED_DURATION: Gauge = register_gauge!( "tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds", "Mean time tasks spent waiting to be polled after being woken for scheduler tokio pool" ) .unwrap(); pub static ref SCHED_TOKIO_POOL_MEAN_POLL_DURATION: Gauge = register_gauge!( "tikv_scheduler_tokio_pool_mean_poll_duration_seconds", "Mean time tasks spent executing (being polled) for scheduler tokio pool" ) .unwrap(); pub static ref SCHED_THROTTLE_TIME: Histogram = register_histogram!( "tikv_scheduler_throttle_duration_seconds", "Bucketed histogram of peer commits logs duration.", exponential_buckets(0.00001, 2.0, 26).unwrap() ).unwrap(); pub static ref SCHED_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_scheduler_command_duration_seconds", "Bucketed histogram of command execution", &["type"], exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref SCHED_HISTOGRAM_VEC_STATIC: SchedDurationVec = auto_flush_from!(SCHED_HISTOGRAM_VEC, SchedDurationVec); pub static ref SCHED_LATCH_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_scheduler_latch_wait_duration_seconds", "Bucketed histogram of latch wait", &["type"], exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref SCHED_LATCH_HISTOGRAM_VEC: SchedLatchDurationVec = auto_flush_from!(SCHED_LATCH_HISTOGRAM, SchedLatchDurationVec); pub static ref SCHED_PROCESSING_READ_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_scheduler_processing_read_duration_seconds", "Bucketed histogram of processing read duration", &["type"], exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref SCHED_PROCESSING_READ_HISTOGRAM_STATIC: ProcessingReadVec = auto_flush_from!(SCHED_PROCESSING_READ_HISTOGRAM_VEC, ProcessingReadVec); pub static ref SCHED_PROCESSING_WRITE_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_scheduler_processing_write_duration_seconds", "Bucketed histogram of processing write duration", &["type"], exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref SCHED_TOO_BUSY_COUNTER: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_too_busy_total", "Total count of scheduler too busy", &["type"] ) .unwrap(); pub static ref SCHED_TOO_BUSY_COUNTER_VEC: SchedTooBusyVec = auto_flush_from!(SCHED_TOO_BUSY_COUNTER, SchedTooBusyVec); pub static ref SCHED_TOO_BUSY_REASON_COUNTER: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_too_busy_reason_total", "Total count of scheduler too busy by reason", &["reason"] ) .unwrap(); pub static ref SCHED_TOO_BUSY_REASON_COUNTER_VEC: SchedTooBusyReasonVec = auto_flush_from!(SCHED_TOO_BUSY_REASON_COUNTER, SchedTooBusyReasonVec); pub static ref SCHED_COMMANDS_PRI_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_commands_pri_total", "Total count of different priority commands", &["priority"] ) .unwrap(); pub static ref SCHED_COMMANDS_PRI_COUNTER_VEC_STATIC: SchedCommandPriCounterVec = auto_flush_from!(SCHED_COMMANDS_PRI_COUNTER_VEC, SchedCommandPriCounterVec); pub static ref KV_COMMAND_KEYREAD_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_scheduler_kv_command_key_read", "Bucketed histogram of keys read of a kv command", &["type"], exponential_buckets(1.0, 2.0, 21).unwrap() ) .unwrap(); pub static ref KV_COMMAND_KEYREAD_HISTOGRAM_STATIC: KReadVec = auto_flush_from!(KV_COMMAND_KEYREAD_HISTOGRAM_VEC, KReadVec); pub static ref KV_COMMAND_SCAN_DETAILS: IntCounterVec = register_int_counter_vec!( "tikv_scheduler_kv_scan_details", "Bucketed counter of kv keys scan details for each cf", &["req", "cf", "tag"] ) .unwrap(); pub static ref KV_COMMAND_SCAN_DETAILS_STATIC: CommandScanDetails = auto_flush_from!(KV_COMMAND_SCAN_DETAILS, CommandScanDetails); pub static ref KV_COMMAND_KEYWRITE_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_scheduler_kv_command_key_write", "Bucketed histogram of keys write of a kv command", &["type"], exponential_buckets(1.0, 2.0, 21).unwrap() ) .unwrap(); pub static ref KV_COMMAND_KEYWRITE_HISTOGRAM_VEC: KvCommandKeysWrittenVec = auto_flush_from!(KV_COMMAND_KEYWRITE_HISTOGRAM, KvCommandKeysWrittenVec); pub static ref REQUEST_EXCEED_BOUND: IntCounter = register_int_counter!( "tikv_request_exceed_bound", "Counter of request exceed bound" ) .unwrap(); pub static ref CHECK_MEM_LOCK_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_storage_check_mem_lock_duration_seconds", "Histogram of the duration of checking memory locks", &["type", "result"], exponential_buckets(1e-6f64, 4f64, 10).unwrap() // 1us ~ 262ms ) .unwrap(); pub static ref CHECK_MEM_LOCK_DURATION_HISTOGRAM_VEC: CheckMemLockHistogramVec = auto_flush_from!(CHECK_MEM_LOCK_DURATION_HISTOGRAM, CheckMemLockHistogramVec); pub static ref TXN_COMMAND_THROTTLE_TIME_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( "tikv_txn_command_throttle_time_total", "Total throttle time (microsecond) of txn commands.", &["type"] ) .unwrap(); pub static ref TXN_COMMAND_THROTTLE_TIME_COUNTER_VEC_STATIC: TxnCommandThrottleTimeCounterVec = auto_flush_from!(TXN_COMMAND_THROTTLE_TIME_COUNTER_VEC, TxnCommandThrottleTimeCounterVec); pub static ref IN_MEMORY_PESSIMISTIC_LOCKING_COUNTER: IntCounterVec = register_int_counter_vec!( "tikv_in_memory_pessimistic_locking", "Count of different types of in-memory pessimistic locking", &["result"] ) .unwrap(); pub static ref IN_MEMORY_PESSIMISTIC_LOCKING_COUNTER_STATIC: InMemoryPessimisticLockingCounter = auto_flush_from!(IN_MEMORY_PESSIMISTIC_LOCKING_COUNTER, InMemoryPessimisticLockingCounter); pub static ref LOCK_WAIT_QUEUE_ENTRIES_GAUGE_VEC: LockWaitQueueEntriesGauge = register_static_int_gauge_vec!( LockWaitQueueEntriesGauge, "tikv_lock_wait_queue_entries_gauge_vec", "Statistics of the lock wait queue's state", &["type"] ) .unwrap(); pub static ref LOCK_UPGRADE_CONFLICT_COUNTER_VEC: LockUpgradeConflictCounter = register_static_int_counter_vec!( LockUpgradeConflictCounter, "tikv_lock_upgrade_conflict_total", "Total number of shared-lock upgrade conflicts by reason", &["reason"] ) .unwrap(); pub static ref SHARED_LOCK_HOLDER_COUNT_HISTOGRAM: Histogram = register_histogram!( "tikv_shared_lock_holder_count", "Holder count observed when writing a non-empty shared-lock wrapper", exponential_buckets(1.0, 2.0, 16).unwrap() ) .unwrap(); pub static ref LOCK_WAIT_QUEUE_LENGTH_HISTOGRAM: Histogram = register_histogram!( "tikv_lock_wait_queue_length", "Statistics of length of queues counted when enqueueing", exponential_buckets(1.0, 2.0, 16).unwrap() ) .unwrap(); pub static ref TXN_FILE_PREWRITE_REJECT_BY_LOCK_LIMIT_COUNTER: IntCounter = register_int_counter!( "tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total", "Total count of txn-file prewrites rejected because a region already has too many txn-file locks", ).unwrap(); pub static ref TXN_FILE_LOCK_REJECT_IN_NORMAL_PATH_COUNTER: IntCounter = register_int_counter!( "tikv_scheduler_txn_file_lock_reject_in_normal_path_total", "Total count of txn-file locks rejected in normal txn command paths", ).unwrap(); pub static ref SCHED_TXN_FILE_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_scheduler_txn_file_duration_seconds", "Bucketed histogram of txn file command execution", &["type", "stage"], exponential_buckets(0.00001, 2.0, 26).unwrap() ) .unwrap(); pub static ref SCHED_TXN_FILE_HISTOGRAM_VEC_STATIC: SchedTxnFileDurationVec = auto_flush_from!(SCHED_TXN_FILE_HISTOGRAM_VEC, SchedTxnFileDurationVec); pub static ref SCHED_TXN_STATUS_CACHE_SIZE: TxnStatusCacheSizeGauge = register_static_int_gauge_vec!( TxnStatusCacheSizeGauge, "tikv_scheduler_txn_status_cache_size", "Statistics of size and capacity of txn status cache (represented in count of entries)", &["type"] ).unwrap(); pub static ref SCHED_BACKGROUND_POOL_RUNNING_TASKS_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_background_pool_running_tasks", "Number of running tasks in the scheduler background pool" ) .unwrap(); pub static ref SCHED_POOL_RUNNING_TASKS_GAUGE: IntGauge = register_int_gauge!( "tikv_scheduler_pool_running_tasks", "Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)" ) .unwrap(); }