// Copyright 2025 PingCAP, Inc. use std::sync::Arc; use dashmap::DashMap; use lazy_static::*; use prometheus::*; lazy_static! { static ref KEYSPACE_LABEL_CACHE: DashMap> = DashMap::new(); pub static ref ACTIVE_KEYSPACE_READ_BYTES: IntGaugeVec = register_int_gauge_vec!( "tikv_active_keyspace_read_bytes", "Total bytes read by active keyspace", &["keyspace_id"] ) .unwrap(); pub static ref REQUEST_WAIT_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( "tikv_request_wait_duration_seconds", "Bucketed histogram of request wait duration", &["type"], exponential_buckets(0.0001, 2.0, 15).unwrap() // 100us ~ 3s ) .unwrap(); // CPU Throttle Metrics pub static ref CPU_THROTTLE_ALLOCATIONS: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_allocations_total", "Total CPU throttle token allocations (success or failure)", &["result"] // result: success, global_exhausted, keyspace_exhausted, fair_share_exceeded ) .unwrap(); pub static ref CPU_THROTTLE_REFILL_RATE_ADJUSTMENTS: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_refill_rate_adjustments_total", "Total refill rate adjustments due to dynamic adjustment", &["level", "direction"] // level: global/keyspace, direction: increase/decrease ) .unwrap(); pub static ref CPU_THROTTLE_GLOBAL_BUCKET_AVAILABLE: IntGauge = register_int_gauge!( "tikv_cpu_throttle_global_bucket_available_us", "Available global CPU budget in microseconds at the current fair-gate charge scale" ) .unwrap(); pub static ref CPU_THROTTLE_GLOBAL_BUCKET_DEBT_US: IntGauge = register_int_gauge!( "tikv_cpu_throttle_global_bucket_debt_us", "Global CPU debt in microseconds at the current fair-gate charge scale" ) .unwrap(); pub static ref CPU_THROTTLE_GLOBAL_BUCKET_CAPACITY: IntGauge = register_int_gauge!( "tikv_cpu_throttle_global_bucket_capacity_us", "Global CPU bucket capacity in microseconds at the current fair-gate charge scale" ) .unwrap(); pub static ref CPU_THROTTLE_GLOBAL_REFILL_RATE: IntGauge = register_int_gauge!( "tikv_cpu_throttle_global_refill_rate_us_per_interval", "Current global CPU refill rate in microseconds per interval at the fair-gate charge scale" ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_BUCKET_AVAILABLE: IntGauge = register_int_gauge!( "tikv_cpu_throttle_low_priority_dag_bucket_available_us", "Available CPU tokens in the low-priority DAG global bucket (microseconds)" ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_BUCKET_CAPACITY: IntGauge = register_int_gauge!( "tikv_cpu_throttle_low_priority_dag_bucket_capacity_us", "Capacity of the low-priority DAG global bucket (microseconds)" ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_REFILL_RATE: IntGauge = register_int_gauge!( "tikv_cpu_throttle_low_priority_dag_refill_rate_us_per_interval", "Current low-priority DAG bucket refill rate (microseconds per interval)" ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_ALLOCATIONS: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_low_priority_dag_allocations_total", "Total low-priority DAG CPU throttle allocation attempts", &["result"] ) .unwrap(); pub static ref CPU_THROTTLE_KEYSPACE_BUCKET_AVAILABLE: IntGaugeVec = register_int_gauge_vec!( "tikv_cpu_throttle_keyspace_bucket_available_us", "Available CPU tokens in keyspace bucket (microseconds)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_KEYSPACE_BUCKET_CAPACITY: IntGaugeVec = register_int_gauge_vec!( "tikv_cpu_throttle_keyspace_bucket_capacity_us", "Capacity of keyspace CPU token bucket (microseconds)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_KEYSPACE_REFILL_RATE: IntGaugeVec = register_int_gauge_vec!( "tikv_cpu_throttle_keyspace_refill_rate_us_per_interval", "Current keyspace CPU token refill rate (microseconds per interval)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_KEYSPACE_INFLIGHT_US: IntGaugeVec = register_int_gauge_vec!( "tikv_cpu_throttle_keyspace_inflight_us", "Inflight CPU tokens allocated per keyspace (microseconds)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_KEYSPACE_DEBT_US: IntGaugeVec = register_int_gauge_vec!( "tikv_cpu_throttle_keyspace_debt_us", "Keyspace CPU token debt when inflight exceeds quota (microseconds)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_TENANT_CAP_RATIO: Gauge = register_gauge!( "tikv_cpu_throttle_tenant_cap_ratio", "Current uniform per-tenant CPU cap ratio (0 means disabled)" ) .unwrap(); /// Multiplier applied to measured CPU before charging the global bucket on /// the fair gate path. 1.0 means no fairness override. Global bucket /// gauges with a `_us` suffix are already converted back to CPU /// microseconds using this scale. pub static ref CPU_THROTTLE_FAIR_CHARGE_SCALE: Gauge = register_gauge!( "tikv_cpu_throttle_fair_charge_scale", "Fair-gate CPU charge scale (1.0 means no fairness read CPU ratio override)" ) .unwrap(); /// The ratio actually enforced. Equals the requested ratio only when a fair /// global CPU gate is registered; otherwise the override is inert and this /// reports the startup ratio. pub static ref CPU_THROTTLE_EFFECTIVE_MAX_READ_CPU_RATIO: Gauge = register_gauge!( "tikv_cpu_throttle_effective_max_read_cpu_ratio", "Read CPU ratio currently in force on the fair gate" ) .unwrap(); /// The ratio requested by config, whether or not it is in force. Compare /// against the effective gauge to spot an override set on a node whose fair /// gate is disabled. pub static ref CPU_THROTTLE_REQUESTED_MAX_READ_CPU_RATIO: Gauge = register_gauge!( "tikv_cpu_throttle_requested_max_read_cpu_ratio", "Read CPU ratio requested by the keyspace-fair read-pool config" ) .unwrap(); pub static ref CPU_THROTTLE_TENANT_CAP_QUOTA_US: IntGauge = register_int_gauge!( "tikv_cpu_throttle_tenant_cap_quota_us", "Computed uniform per-tenant CPU cap quota in microseconds per refill interval" ) .unwrap(); pub static ref CPU_THROTTLE_FAIR_ALLOCATIONS: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_fair_allocations_total", "Total fair allocation triggers when global CPU > threshold", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_BURST_COUNT: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_burst_total", "Total burst allocations when keyspace quota exhausted but global available", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_BURST_TOKENS_USED: IntCounterVec = register_int_counter_vec!( "tikv_cpu_throttle_burst_tokens_used_total", "Total CPU tokens used in burst mode (in microseconds)", &["keyspace_id"] ) .unwrap(); pub static ref CPU_TOKEN_WAIT_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_token_wait_duration_seconds", "Histogram of CPU token allocation wait duration", &["keyspace_id", "result"] // result: success, timeout ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_CPU_TOKEN_WAIT_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds", "Histogram of low-priority DAG CPU token allocation wait duration", &["result"] ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_WAIT_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_throttle_low_priority_dag_wait_duration_seconds", "Histogram of total low-priority DAG wait duration inside the read pool", &["result"] ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_KEYSPACE_WAIT_DURATION_HISTOGRAM: Histogram = register_histogram!( "tikv_cpu_throttle_low_priority_dag_keyspace_wait_duration_seconds", "Histogram of low-priority DAG keyspace wait duration" ) .unwrap(); pub static ref CPU_TOKEN_RUNTIME_WAIT_DURATION_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_token_runtime_wait_duration_seconds", "Histogram of runtime CPU token allocation wait duration", &["keyspace_id", "result"] // result: success, timeout ) .unwrap(); pub static ref CPU_TOKEN_RUNTIME_ALLOCATIONS: IntCounterVec = register_int_counter_vec!( "tikv_cpu_token_runtime_allocations_total", "Total runtime CPU token allocations", &["result"] // result: success, timeout ) .unwrap(); pub static ref CPU_TOKEN_RETRY_COUNT: IntCounterVec = register_int_counter_vec!( "tikv_cpu_token_allocation_retries_total", "Total number of retries during CPU token allocation", &["keyspace_id"] ) .unwrap(); pub static ref CPU_USAGE_MONITOR_GLOBAL: prometheus::Gauge = register_gauge!( "tikv_cpu_usage_monitor_global_ratio", "Global CPU usage ratio (0.0 - 1.0)" ) .unwrap(); pub static ref CPU_USAGE_MONITOR_KEYSPACE: prometheus::GaugeVec = register_gauge_vec!( "tikv_cpu_usage_monitor_keyspace_ratio", "Per-keyspace CPU usage ratio (0.0 - 1.0)", &["keyspace_id"] ) .unwrap(); pub static ref RESOURCE_CONTROL_GLOBAL_USAGE_SAMPLES: IntGauge = register_int_gauge!( "tikv_resource_control_global_usage_samples", "Current number of global resource control usage samples kept in memory" ) .unwrap(); pub static ref RESOURCE_CONTROL_KEYSPACE_USAGE_SAMPLES: IntGauge = register_int_gauge!( "tikv_resource_control_keyspace_usage_samples", "Current number of keyspace resource control usage samples kept in memory" ) .unwrap(); pub static ref RESOURCE_CONTROL_TICKS_TOTAL: IntCounter = register_int_counter!( "tikv_resource_control_ticks_total", "Total number of resource control event hub ticks" ) .unwrap(); pub static ref CPU_THROTTLE_REQUEST_CPU_TIME_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_throttle_request_cpu_time_seconds", "Histogram of best-available CPU time per throttled request", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_REQUEST_DAG_MAIN_CPU_TIME_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_throttle_request_dag_main_cpu_time_seconds", "Histogram of DAG main execution CPU time per throttled request", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_REQUEST_ACTUAL_TO_ESTIMATED_RATIO_HISTOGRAM: HistogramVec = register_histogram_vec!( "tikv_cpu_throttle_request_actual_to_estimated_ratio", "Histogram of DAG main execution CPU time to estimated ratio per request", &["keyspace_id"] ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_REQUEST_CPU_TIME_HISTOGRAM: Histogram = register_histogram!( "tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds", "Histogram of CPU time per low-priority DAG request" ) .unwrap(); pub static ref CPU_THROTTLE_LOW_PRIORITY_DAG_REQUEST_ACTUAL_TO_ESTIMATED_RATIO_HISTOGRAM: Histogram = register_histogram!( "tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio", "Histogram of actual CPU time to estimated ratio for low-priority DAG requests" ) .unwrap(); // PD service_limit fetcher metrics pub static ref SERVICE_LIMIT_FETCH_TOTAL: IntCounterVec = register_int_counter_vec!( "tikv_service_limit_fetch_total", "Total PD service_limit fetch attempts", &["result"] // result: success, error ) .unwrap(); pub static ref SERVICE_LIMIT_FETCH_DURATION: Histogram = register_histogram!( "tikv_service_limit_fetch_duration_seconds", "Histogram of PD service_limit fetch duration" ) .unwrap(); pub static ref SERVICE_LIMIT_LAST_SUCCESS_TIMESTAMP: Gauge = register_gauge!( "tikv_service_limit_last_success_timestamp", "Unix timestamp of last successful service_limit fetch" ) .unwrap(); pub static ref SERVICE_LIMIT_KEYSPACE_VALUE: GaugeVec = register_gauge_vec!( "tikv_service_limit_keyspace_value", "Current keyspace service_limit value (RU/s)", &["keyspace_id"] ) .unwrap(); pub static ref SERVICE_LIMIT_KEYSPACE_QUOTA_US: IntGaugeVec = register_int_gauge_vec!( "tikv_service_limit_keyspace_quota_us", "CPU quota derived from PD service_limit (microseconds per interval)", &["keyspace_id"] ) .unwrap(); } pub(crate) fn keyspace_label(keyspace_id: u32) -> Arc { if let Some(entry) = KEYSPACE_LABEL_CACHE.get(&keyspace_id) { return Arc::clone(entry.value()); } let label: Arc = Arc::from(keyspace_id.to_string()); let entry = KEYSPACE_LABEL_CACHE .entry(keyspace_id) .or_insert_with(|| Arc::clone(&label)); Arc::clone(entry.value()) } pub(crate) fn remove_keyspace_label(keyspace_id: u32) { KEYSPACE_LABEL_CACHE.remove(&keyspace_id); }