use std::time::Duration; use lazy_static::lazy_static; use prometheus::{ HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, exponential_buckets, }; use tici_common::metrics::{ OperationMetrics, OperationMetricsGuard, begin_operation_metrics1, register_histogram_with_labels, register_histogram_with_labels_and_buckets, register_int_counter_with_labels, register_int_gauge, register_int_gauge_with_labels, }; // Doc cache loads are normally sub‑millisecond; default Prometheus buckets // start at 5ms, so use `exponential_buckets(100µs, ×2, 15)` instead. // // Histogram upper bounds (`le`, seconds), then millis for readability. // Samples above the last finite bucket accumulate in `+Inf`. // // | # | upper bound (s) | upper bound | // |---|-----------------|-------------| // | 1 | 0.0001 | 100µs | // | 2 | 0.0002 | 200µs | // | 3 | 0.0004 | 400µs | // | 4 | 0.0008 | 800µs | // | 5 | 0.0016 | 1.6ms | // | 6 | 0.0032 | 3.2ms | // | 7 | 0.0064 | 6.4ms | // | 8 | 0.0128 | 12.8ms | // | 9 | 0.0256 | 25.6ms | // | 10| 0.0512 | 51.2ms | // | 11| 0.1024 | 102.4ms | // | 12| 0.2048 | 204.8ms | // | 13| 0.4096 | 409.6ms | // | 14| 0.8192 | 819.2ms | // | 15| 1.6384 | 1638.4ms | fn doc_cache_load_latency_buckets() -> Vec { exponential_buckets(0.0001, 2.0, 15) .unwrap_or_else(|err| panic!("doc cache load latency histogram buckets invalid: {err}")) } // Shard query open/next measure active processing time from 1ms up to tens of // seconds. Default Prometheus buckets start at 5ms and lose resolution on fast // paths. fn shard_query_active_duration_buckets() -> Vec { exponential_buckets(0.001, 2.0, 16) .unwrap_or_else(|err| panic!("shard query active duration buckets invalid: {err}")) } fn shard_query_batch_count_buckets() -> Vec { vec![ 0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, ] } lazy_static! { pub static ref READER_SHARD_GAUGE: IntGauge = register_int_gauge("tici_reader_shard_count", "Number of shards"); pub static ref READER_FRAG_GAUGE: IntGauge = register_int_gauge("tici_reader_frag_count", "Number of fragments"); pub static ref READER_QUERY_LATENCY: HistogramVec = register_histogram_with_labels( "tici_reader_query_latency", "Latency of queries", &["type"], ); pub static ref READER_SHARD_QUERY_DURATION_HISTOGRAM: HistogramVec = register_histogram_with_labels( "tici_reader_shard_query_duration", "Wall-clock duration of shard reader search streams from open to drop", &["type"], ); pub static ref READER_SHARD_QUERY_OPEN_DURATION_HISTOGRAM: HistogramVec = register_histogram_with_labels_and_buckets( "tici_reader_shard_query_open_duration", "Active processing time spent opening a shard reader search stream", &["type"], shard_query_active_duration_buckets(), ); pub static ref READER_SHARD_QUERY_NEXT_DURATION_HISTOGRAM: HistogramVec = register_histogram_with_labels_and_buckets( "tici_reader_shard_query_next_duration", "Total active processing time spent in next_batch across a shard reader search stream", &["type"], shard_query_active_duration_buckets(), ); pub static ref READER_SHARD_QUERY_BATCH_COUNT_HISTOGRAM: HistogramVec = register_histogram_with_labels_and_buckets( "tici_reader_shard_query_batch_count", "Number of non-empty next_batch calls per shard reader search stream", &["type"], shard_query_batch_count_buckets(), ); pub static ref READER_SHARD_QUERY_COUNTER: IntCounterVec = register_int_counter_with_labels( "tici_reader_shard_query_count", "The count of shard reader queries", &["type", "result"], ); pub static ref READER_SHARD_QUERY_INFLIGHT_GAUGE: IntGaugeVec = register_int_gauge_with_labels( "tici_reader_shard_query_inflight", "The inflight count of shard reader queries", &["type"], ); static ref READER_DOC_CACHE_ACCESS_COUNTER: IntCounterVec = register_int_counter_with_labels( "tici_reader_doc_cache_access_count", "Number of doc cache accesses (result: resident_hit = value already cached; \ load_miss = this thread loaded; coalesced_wait = waited for in-flight load)", &["result"], ); static ref READER_DOC_CACHE_RESIDENT_HIT_COUNTER: IntCounter = READER_DOC_CACHE_ACCESS_COUNTER.with_label_values(&["resident_hit"]); static ref READER_DOC_CACHE_LOAD_MISS_COUNTER: IntCounter = READER_DOC_CACHE_ACCESS_COUNTER.with_label_values(&["load_miss"]); static ref READER_DOC_CACHE_COALESCED_WAIT_COUNTER: IntCounter = READER_DOC_CACHE_ACCESS_COUNTER.with_label_values(&["coalesced_wait"]); static ref READER_DOC_CACHE_LOAD_LATENCY: HistogramVec = register_histogram_with_labels_and_buckets( "tici_reader_doc_cache_load_latency", "Latency of doc cache loads", &["result"], doc_cache_load_latency_buckets(), ); } pub fn begin_shard_reader_query_metrics(query_type: &'static str) -> OperationMetricsGuard { begin_operation_metrics1( OperationMetrics::new( &READER_SHARD_QUERY_DURATION_HISTOGRAM, &READER_SHARD_QUERY_COUNTER, &READER_SHARD_QUERY_INFLIGHT_GAUGE, ), query_type, ) } pub fn observe_shard_search_stream_timing( query_type: &'static str, open_us: u64, next_batch_us: u64, batch_count: u64, ) { READER_SHARD_QUERY_OPEN_DURATION_HISTOGRAM .with_label_values(&[query_type]) .observe(open_us as f64 / 1_000_000.0); READER_SHARD_QUERY_NEXT_DURATION_HISTOGRAM .with_label_values(&[query_type]) .observe(next_batch_us as f64 / 1_000_000.0); READER_SHARD_QUERY_BATCH_COUNT_HISTOGRAM .with_label_values(&[query_type]) .observe(batch_count as f64); } pub fn inc_doc_cache_resident_hit_count() { READER_DOC_CACHE_RESIDENT_HIT_COUNTER.inc(); } pub fn inc_doc_cache_load_miss_count() { READER_DOC_CACHE_LOAD_MISS_COUNTER.inc(); } pub fn inc_doc_cache_coalesced_wait_count() { READER_DOC_CACHE_COALESCED_WAIT_COUNTER.inc(); } pub fn observe_doc_cache_load_latency(result: &'static str, latency: Duration) { READER_DOC_CACHE_LOAD_LATENCY .with_label_values(&[result]) .observe(latency.as_secs_f64()); }