package runtimeworker import ( "database/sql" "time" "github.com/prometheus/client_golang/prometheus" ) type workerMetricsSnapshot struct { QueueDepth int64 PoisonDepth int64 OldestAge time.Duration InFlight int64 Attempts uint64 Failures uint64 Processed uint64 WorkLoop workLoopSnapshot ScheduledFlow scheduledFlowMetricsSnapshot PollActivations uint64 SuccessorActivations uint64 PollWakeCoalesced uint64 SuccessorWakeCoalesced uint64 PollFirstEmptyAttempts uint64 SuccessorFirstEmptyAttempts uint64 ContinuationAttempts uint64 ContinuationEmptyAttempts uint64 AdmissionFenceAttempts uint64 AdmissionFenceDuration time.Duration ObservationEnabled bool ObservationLastSuccess time.Time ObservationDuration time.Duration ObservationFailures uint64 ObservationStale bool OldestObservedAt time.Time } type scheduledFlowMetricsSnapshot struct { Enabled bool DiscoveryPasses uint64 DefinitionsConsidered uint64 OccurrencesAccepted uint64 RecoveryScanned uint64 RecoveryRecovered uint64 ExecutionStarted uint64 ExecutionCompleted uint64 ExecutionFailed uint64 CapacityDeferrals uint64 DiscoveryFailures uint64 DiscoveryDuration time.Duration RecoveryDuration time.Duration ExecutionDuration time.Duration } type workerSnapshotFunc func() workerMetricsSnapshot type workerMetricSources struct { WorkFamily string ExecutionClass string Snapshot workerSnapshotFunc DB *sql.DB Startup *workerStartupObservation Permission func() authorityCheckSnapshot Ownership func() authorityCheckSnapshot Database workerDatabaseObservations } type metricsCollector struct { sources workerMetricSources info, startupDuration, startupStageDuration *prometheus.Desc authorityChecks, authorityFailures, authorityDuration, authorityLastSuccess *prometheus.Desc queueDepth, poisonDepth, oldestAge, inFlight, attempts, failures, processed *prometheus.Desc pollChecks, confirmedIdle, hintFailures, claimAttempts, emptyClaims, claimed *prometheus.Desc pollDuration, claimDuration *prometheus.Desc pollActivations, successorActivations, pollWakeCoalesced, successorWakeCoalesced *prometheus.Desc pollFirstEmpty, successorFirstEmpty, continuationAttempts, continuationEmpty *prometheus.Desc admissionFenceAttempts, admissionFenceSeconds *prometheus.Desc oldestObservedAt, observationLastSuccess, observationDuration *prometheus.Desc observationFailures, observationStale *prometheus.Desc poolMaxOpen, poolOpen, poolInUse, poolIdle, poolWaitCount *prometheus.Desc poolWaitSeconds *prometheus.Desc auxPoolActive, auxPoolMaxOpen, auxPoolOpen, auxPoolInUse, auxPoolIdle *prometheus.Desc auxPoolWaitCount, auxPoolWaitSeconds, auxPoolOpens, auxPoolOpenFailures *prometheus.Desc auxPoolCloses, transientOpens, transientOpenFailures, transientOpenDuration *prometheus.Desc scheduledDiscovery, scheduledDefinitions, scheduledAccepted, scheduledRecoveryScanned *prometheus.Desc scheduledRecoveryRecovered, scheduledExecutionStarted, scheduledExecutionCompleted *prometheus.Desc scheduledExecutionFailed, scheduledCapacityDeferrals, scheduledDiscoveryFailures *prometheus.Desc scheduledPhaseDuration *prometheus.Desc } func newMetricsCollector(sources workerMetricSources) *metricsCollector { return &metricsCollector{ sources: sources, info: prometheus.NewDesc("nexuscrm_runtime_worker_info", "Immutable Runtime Worker role identity.", []string{"work_family", "execution_class"}, nil), startupDuration: prometheus.NewDesc("nexuscrm_runtime_worker_startup_duration_seconds", "Wall time from Worker assembly entry to completed process assembly.", nil, nil), startupStageDuration: prometheus.NewDesc("nexuscrm_runtime_worker_startup_stage_duration_seconds", "Cumulative Worker assembly time attributed to a fixed startup stage.", []string{"stage"}, nil), authorityChecks: prometheus.NewDesc("nexuscrm_runtime_worker_authority_checks_total", "Active durable authority checks completed by this Worker.", []string{"authority"}, nil), authorityFailures: prometheus.NewDesc("nexuscrm_runtime_worker_authority_check_failures_total", "Durable authority checks that rejected readiness or returned an error.", []string{"authority"}, nil), authorityDuration: prometheus.NewDesc("nexuscrm_runtime_worker_authority_check_duration_seconds_total", "Cumulative wall time spent in active durable authority checks.", []string{"authority"}, nil), authorityLastSuccess: prometheus.NewDesc("nexuscrm_runtime_worker_authority_last_success_timestamp_seconds", "Unix timestamp of the latest successful durable authority check; zero until first success.", []string{"authority"}, nil), queueDepth: prometheus.NewDesc("nexuscrm_runtime_worker_queue_depth", "Latest durable pending work count.", nil, nil), poisonDepth: prometheus.NewDesc("nexuscrm_runtime_worker_poison_depth", "Latest durable work count requiring operator repair.", nil, nil), oldestAge: prometheus.NewDesc("nexuscrm_runtime_worker_oldest_age_seconds", "Age of the oldest pending work item at the latest successful durable queue observation.", nil, nil), inFlight: prometheus.NewDesc("nexuscrm_runtime_worker_in_flight", "Work items currently executing in this process.", nil, nil), attempts: prometheus.NewDesc("nexuscrm_runtime_worker_attempts_total", "Work attempts started by this process.", nil, nil), failures: prometheus.NewDesc("nexuscrm_runtime_worker_failures_total", "Work attempts that encountered a processing or finalization failure.", nil, nil), processed: prometheus.NewDesc("nexuscrm_runtime_worker_processed_total", "Work items finalized successfully by this process.", nil, nil), pollChecks: prometheus.NewDesc("nexuscrm_runtime_worker_poll_checks_total", "Bounded admission hint or discovery checks completed by this Worker.", nil, nil), confirmedIdle: prometheus.NewDesc("nexuscrm_runtime_worker_confirmed_idle_checks_total", "Successful admission checks that confirmed no immediately eligible work.", nil, nil), hintFailures: prometheus.NewDesc("nexuscrm_runtime_worker_hint_failures_total", "Admission hint checks that failed and therefore retained the authoritative work-attempt path.", nil, nil), claimAttempts: prometheus.NewDesc("nexuscrm_runtime_worker_claim_attempts_total", "Authoritative durable work attempts completed by this Worker.", nil, nil), emptyClaims: prometheus.NewDesc("nexuscrm_runtime_worker_empty_claim_attempts_total", "Successful authoritative work attempts that found no claimable row.", nil, nil), claimed: prometheus.NewDesc("nexuscrm_runtime_worker_claimed_total", "Durable work rows claimed or otherwise admitted for execution.", nil, nil), pollDuration: prometheus.NewDesc("nexuscrm_runtime_worker_poll_check_duration_seconds_total", "Cumulative wall time spent in bounded admission hint or discovery checks.", nil, nil), claimDuration: prometheus.NewDesc("nexuscrm_runtime_worker_claim_duration_seconds_total", "Cumulative wall time spent in authoritative claim or combined claim-and-effect attempts.", nil, nil), pollActivations: prometheus.NewDesc("nexuscrm_runtime_worker_poll_activations_total", "Parallel drain passes activated by the bounded poll coordinator.", nil, nil), successorActivations: prometheus.NewDesc("nexuscrm_runtime_worker_successor_activations_total", "Parallel drain passes activated by a successful predecessor.", nil, nil), pollWakeCoalesced: prometheus.NewDesc("nexuscrm_runtime_worker_poll_wake_coalesced_total", "Poll wake submissions coalesced because one wake was already queued.", nil, nil), successorWakeCoalesced: prometheus.NewDesc("nexuscrm_runtime_worker_successor_wake_coalesced_total", "Successor wake submissions coalesced because one wake was already queued.", nil, nil), pollFirstEmpty: prometheus.NewDesc("nexuscrm_runtime_worker_poll_first_empty_attempts_total", "Poll-activated drain passes whose first fenced work attempt found no claimable row.", nil, nil), successorFirstEmpty: prometheus.NewDesc("nexuscrm_runtime_worker_successor_first_empty_attempts_total", "Successor-activated drain passes whose first fenced work attempt found no claimable row.", nil, nil), continuationAttempts: prometheus.NewDesc("nexuscrm_runtime_worker_continuation_attempts_total", "Additional fenced work attempts made by an already active drain pass.", nil, nil), continuationEmpty: prometheus.NewDesc("nexuscrm_runtime_worker_continuation_empty_attempts_total", "Additional attempts by an already active drain pass that found no claimable row.", nil, nil), admissionFenceAttempts: prometheus.NewDesc("nexuscrm_runtime_worker_admission_fence_attempts_total", "Durable admission ownership-fence attempts made by this worker.", nil, nil), admissionFenceSeconds: prometheus.NewDesc("nexuscrm_runtime_worker_admission_fence_duration_seconds_total", "Cumulative wall time spent acquiring and validating the durable admission ownership fence.", nil, nil), oldestObservedAt: prometheus.NewDesc("nexuscrm_runtime_worker_oldest_active_timestamp_seconds", "Unix timestamp of the oldest pending or processing durable row in the latest successful queue observation; zero when none.", nil, nil), observationLastSuccess: prometheus.NewDesc("nexuscrm_runtime_worker_queue_observation_last_success_timestamp_seconds", "Unix timestamp of the last successful durable queue observation.", nil, nil), observationDuration: prometheus.NewDesc("nexuscrm_runtime_worker_queue_observation_duration_seconds", "Wall time of the latest durable queue observation attempt.", nil, nil), observationFailures: prometheus.NewDesc("nexuscrm_runtime_worker_queue_observation_failures_total", "Durable queue observation attempts that failed or timed out since process start.", nil, nil), observationStale: prometheus.NewDesc("nexuscrm_runtime_worker_queue_observation_stale", "Whether the cached durable queue observation has never succeeded or the latest attempt failed.", nil, nil), poolMaxOpen: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_max_open_connections", "Configured worker database pool maximum.", nil, nil), poolOpen: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_open_connections", "Current open worker database connections.", nil, nil), poolInUse: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_in_use_connections", "Current in-use worker database connections.", nil, nil), poolIdle: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_idle_connections", "Current idle worker database connections.", nil, nil), poolWaitCount: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_wait_count_total", "Worker database pool waits since process start.", nil, nil), poolWaitSeconds: prometheus.NewDesc("nexuscrm_runtime_worker_db_pool_wait_duration_seconds_total", "Worker database pool wait time since process start.", nil, nil), auxPoolActive: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_active", "Whether the purpose-owned long-lived auxiliary database pool is currently active.", []string{"purpose"}, nil), auxPoolMaxOpen: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_max_open_connections", "Configured maximum for a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolOpen: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_open_connections", "Current open connections in a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolInUse: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_in_use_connections", "Current in-use connections in a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolIdle: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_idle_connections", "Current idle connections in a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolWaitCount: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_wait_count_total", "Connection waits in a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolWaitSeconds: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_wait_duration_seconds_total", "Connection wait time in a purpose-owned long-lived auxiliary database pool.", []string{"purpose"}, nil), auxPoolOpens: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_open_attempts_total", "Purpose-owned long-lived auxiliary database pool open attempts.", []string{"purpose"}, nil), auxPoolOpenFailures: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_open_failures_total", "Failed purpose-owned long-lived auxiliary database pool open attempts.", []string{"purpose"}, nil), auxPoolCloses: prometheus.NewDesc("nexuscrm_runtime_worker_auxiliary_db_pool_closes_total", "Purpose-owned long-lived auxiliary database pool closes.", []string{"purpose"}, nil), transientOpens: prometheus.NewDesc("nexuscrm_runtime_worker_transient_db_open_attempts_total", "Transient database pool open attempts.", []string{"purpose"}, nil), transientOpenFailures: prometheus.NewDesc("nexuscrm_runtime_worker_transient_db_open_failures_total", "Failed transient database pool open attempts.", []string{"purpose"}, nil), transientOpenDuration: prometheus.NewDesc("nexuscrm_runtime_worker_transient_db_open_duration_seconds_total", "Cumulative transient database pool open time.", []string{"purpose"}, nil), scheduledDiscovery: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_discovery_passes_total", "Scheduled Flow discovery passes completed by this Worker.", nil, nil), scheduledDefinitions: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_definitions_considered_total", "Scheduled Flow definitions considered by discovery.", nil, nil), scheduledAccepted: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_occurrences_accepted_total", "Durable scheduled Flow runs accepted for execution.", nil, nil), scheduledRecoveryScanned: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_recovery_scanned_total", "Durable scheduled Flow recovery candidates scanned.", nil, nil), scheduledRecoveryRecovered: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_recovery_recovered_total", "Durable scheduled Flow runs successfully reclaimed.", nil, nil), scheduledExecutionStarted: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_execution_started_total", "Durable scheduled Flow executions started.", nil, nil), scheduledExecutionCompleted: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_execution_completed_total", "Durable scheduled Flow executions reaching a terminal outcome.", nil, nil), scheduledExecutionFailed: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_execution_failed_total", "Durable scheduled Flow executions reaching a failed terminal outcome.", nil, nil), scheduledCapacityDeferrals: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_capacity_deferrals_total", "Scheduled Flow work retained durably because the execution lane was full.", nil, nil), scheduledDiscoveryFailures: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_discovery_failures_total", "Scheduled Flow discovery passes that could not advance completely.", nil, nil), scheduledPhaseDuration: prometheus.NewDesc("nexuscrm_runtime_worker_scheduled_phase_duration_seconds_total", "Cumulative Scheduled Flow Worker time by fixed phase.", []string{"phase"}, nil), } } func newWorkerMetricsCollector( workFamily, executionClass string, snapshot workerSnapshotFunc, db *sql.DB, startup *workerStartupObservation, permissionState *observedWorkerPermissionStateGate, ownership *ownershipController, databaseObservations workerDatabaseObservations, ) *metricsCollector { return newMetricsCollector(workerMetricSources{ WorkFamily: workFamily, ExecutionClass: executionClass, Snapshot: snapshot, DB: db, Startup: startup, Permission: permissionState.snapshot, Ownership: ownership.snapshot, Database: databaseObservations, }) } func (m *metricsCollector) Describe(ch chan<- *prometheus.Desc) { for _, descriptor := range []*prometheus.Desc{ m.info, m.startupDuration, m.startupStageDuration, m.authorityChecks, m.authorityFailures, m.authorityDuration, m.authorityLastSuccess, m.queueDepth, m.poisonDepth, m.oldestAge, m.inFlight, m.attempts, m.failures, m.processed, m.pollChecks, m.confirmedIdle, m.hintFailures, m.claimAttempts, m.emptyClaims, m.claimed, m.pollDuration, m.claimDuration, m.pollActivations, m.successorActivations, m.pollWakeCoalesced, m.successorWakeCoalesced, m.pollFirstEmpty, m.successorFirstEmpty, m.continuationAttempts, m.continuationEmpty, m.admissionFenceAttempts, m.admissionFenceSeconds, m.oldestObservedAt, m.observationLastSuccess, m.observationDuration, m.observationFailures, m.observationStale, m.poolMaxOpen, m.poolOpen, m.poolInUse, m.poolIdle, m.poolWaitCount, m.poolWaitSeconds, m.auxPoolActive, m.auxPoolMaxOpen, m.auxPoolOpen, m.auxPoolInUse, m.auxPoolIdle, m.auxPoolWaitCount, m.auxPoolWaitSeconds, m.auxPoolOpens, m.auxPoolOpenFailures, m.auxPoolCloses, m.transientOpens, m.transientOpenFailures, m.transientOpenDuration, m.scheduledDiscovery, m.scheduledDefinitions, m.scheduledAccepted, m.scheduledRecoveryScanned, m.scheduledRecoveryRecovered, m.scheduledExecutionStarted, m.scheduledExecutionCompleted, m.scheduledExecutionFailed, m.scheduledCapacityDeferrals, m.scheduledDiscoveryFailures, m.scheduledPhaseDuration, } { ch <- descriptor } } func (m *metricsCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(m.info, prometheus.GaugeValue, 1, m.sources.WorkFamily, m.sources.ExecutionClass) startup := m.sources.Startup.snapshot() ch <- prometheus.MustNewConstMetric(m.startupDuration, prometheus.GaugeValue, startup.Duration.Seconds()) for _, stage := range workerStartupStages { ch <- prometheus.MustNewConstMetric(m.startupStageDuration, prometheus.GaugeValue, startup.Stages[stage].Seconds(), stage) } m.collectAuthority(ch, "permission_state", m.sources.Permission) m.collectAuthority(ch, "work_ownership", m.sources.Ownership) snapshot := workerMetricsSnapshot{} if m.sources.Snapshot != nil { snapshot = m.sources.Snapshot() } ch <- prometheus.MustNewConstMetric(m.queueDepth, prometheus.GaugeValue, float64(snapshot.QueueDepth)) ch <- prometheus.MustNewConstMetric(m.poisonDepth, prometheus.GaugeValue, float64(snapshot.PoisonDepth)) ch <- prometheus.MustNewConstMetric(m.oldestAge, prometheus.GaugeValue, snapshot.OldestAge.Seconds()) ch <- prometheus.MustNewConstMetric(m.inFlight, prometheus.GaugeValue, float64(snapshot.InFlight)) ch <- prometheus.MustNewConstMetric(m.attempts, prometheus.CounterValue, float64(snapshot.Attempts)) ch <- prometheus.MustNewConstMetric(m.failures, prometheus.CounterValue, float64(snapshot.Failures)) ch <- prometheus.MustNewConstMetric(m.processed, prometheus.CounterValue, float64(snapshot.Processed)) ch <- prometheus.MustNewConstMetric(m.pollChecks, prometheus.CounterValue, float64(snapshot.WorkLoop.PollChecks)) ch <- prometheus.MustNewConstMetric(m.confirmedIdle, prometheus.CounterValue, float64(snapshot.WorkLoop.ConfirmedIdle)) ch <- prometheus.MustNewConstMetric(m.hintFailures, prometheus.CounterValue, float64(snapshot.WorkLoop.HintFailures)) ch <- prometheus.MustNewConstMetric(m.claimAttempts, prometheus.CounterValue, float64(snapshot.WorkLoop.ClaimAttempts)) ch <- prometheus.MustNewConstMetric(m.emptyClaims, prometheus.CounterValue, float64(snapshot.WorkLoop.EmptyClaims)) ch <- prometheus.MustNewConstMetric(m.claimed, prometheus.CounterValue, float64(snapshot.WorkLoop.Claimed)) ch <- prometheus.MustNewConstMetric(m.pollDuration, prometheus.CounterValue, snapshot.WorkLoop.PollDuration.Seconds()) ch <- prometheus.MustNewConstMetric(m.claimDuration, prometheus.CounterValue, snapshot.WorkLoop.ClaimDuration.Seconds()) ch <- prometheus.MustNewConstMetric(m.pollActivations, prometheus.CounterValue, float64(snapshot.PollActivations)) ch <- prometheus.MustNewConstMetric(m.successorActivations, prometheus.CounterValue, float64(snapshot.SuccessorActivations)) ch <- prometheus.MustNewConstMetric(m.pollWakeCoalesced, prometheus.CounterValue, float64(snapshot.PollWakeCoalesced)) ch <- prometheus.MustNewConstMetric(m.successorWakeCoalesced, prometheus.CounterValue, float64(snapshot.SuccessorWakeCoalesced)) ch <- prometheus.MustNewConstMetric(m.pollFirstEmpty, prometheus.CounterValue, float64(snapshot.PollFirstEmptyAttempts)) ch <- prometheus.MustNewConstMetric(m.successorFirstEmpty, prometheus.CounterValue, float64(snapshot.SuccessorFirstEmptyAttempts)) ch <- prometheus.MustNewConstMetric(m.continuationAttempts, prometheus.CounterValue, float64(snapshot.ContinuationAttempts)) ch <- prometheus.MustNewConstMetric(m.continuationEmpty, prometheus.CounterValue, float64(snapshot.ContinuationEmptyAttempts)) ch <- prometheus.MustNewConstMetric(m.admissionFenceAttempts, prometheus.CounterValue, float64(snapshot.AdmissionFenceAttempts)) ch <- prometheus.MustNewConstMetric(m.admissionFenceSeconds, prometheus.CounterValue, snapshot.AdmissionFenceDuration.Seconds()) if snapshot.ObservationEnabled { ch <- prometheus.MustNewConstMetric(m.oldestObservedAt, prometheus.GaugeValue, unixSeconds(snapshot.OldestObservedAt)) ch <- prometheus.MustNewConstMetric(m.observationLastSuccess, prometheus.GaugeValue, unixSeconds(snapshot.ObservationLastSuccess)) ch <- prometheus.MustNewConstMetric(m.observationDuration, prometheus.GaugeValue, snapshot.ObservationDuration.Seconds()) ch <- prometheus.MustNewConstMetric(m.observationFailures, prometheus.CounterValue, float64(snapshot.ObservationFailures)) stale := 0.0 if snapshot.ObservationStale { stale = 1 } ch <- prometheus.MustNewConstMetric(m.observationStale, prometheus.GaugeValue, stale) } stats := m.sources.DB.Stats() ch <- prometheus.MustNewConstMetric(m.poolMaxOpen, prometheus.GaugeValue, float64(stats.MaxOpenConnections)) ch <- prometheus.MustNewConstMetric(m.poolOpen, prometheus.GaugeValue, float64(stats.OpenConnections)) ch <- prometheus.MustNewConstMetric(m.poolInUse, prometheus.GaugeValue, float64(stats.InUse)) ch <- prometheus.MustNewConstMetric(m.poolIdle, prometheus.GaugeValue, float64(stats.Idle)) ch <- prometheus.MustNewConstMetric(m.poolWaitCount, prometheus.CounterValue, float64(stats.WaitCount)) ch <- prometheus.MustNewConstMetric(m.poolWaitSeconds, prometheus.CounterValue, stats.WaitDuration.Seconds()) for _, pool := range m.sources.Database.poolSnapshots() { ch <- prometheus.MustNewConstMetric(m.auxPoolActive, prometheus.GaugeValue, boolMetric(pool.Active), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolMaxOpen, prometheus.GaugeValue, float64(pool.Stats.MaxOpenConnections), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolOpen, prometheus.GaugeValue, float64(pool.Stats.OpenConnections), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolInUse, prometheus.GaugeValue, float64(pool.Stats.InUse), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolIdle, prometheus.GaugeValue, float64(pool.Stats.Idle), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolWaitCount, prometheus.CounterValue, float64(pool.Stats.WaitCount), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolWaitSeconds, prometheus.CounterValue, pool.Stats.WaitDuration.Seconds(), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolOpens, prometheus.CounterValue, float64(pool.OpenAttempts), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolOpenFailures, prometheus.CounterValue, float64(pool.OpenFailures), pool.Purpose) ch <- prometheus.MustNewConstMetric(m.auxPoolCloses, prometheus.CounterValue, float64(pool.Closes), pool.Purpose) } for _, transient := range m.sources.Database.transientSnapshots() { ch <- prometheus.MustNewConstMetric(m.transientOpens, prometheus.CounterValue, float64(transient.OpenAttempts), transient.Purpose) ch <- prometheus.MustNewConstMetric(m.transientOpenFailures, prometheus.CounterValue, float64(transient.OpenFailures), transient.Purpose) ch <- prometheus.MustNewConstMetric(m.transientOpenDuration, prometheus.CounterValue, transient.OpenDuration.Seconds(), transient.Purpose) } if scheduled := snapshot.ScheduledFlow; scheduled.Enabled { ch <- prometheus.MustNewConstMetric(m.scheduledDiscovery, prometheus.CounterValue, float64(scheduled.DiscoveryPasses)) ch <- prometheus.MustNewConstMetric(m.scheduledDefinitions, prometheus.CounterValue, float64(scheduled.DefinitionsConsidered)) ch <- prometheus.MustNewConstMetric(m.scheduledAccepted, prometheus.CounterValue, float64(scheduled.OccurrencesAccepted)) ch <- prometheus.MustNewConstMetric(m.scheduledRecoveryScanned, prometheus.CounterValue, float64(scheduled.RecoveryScanned)) ch <- prometheus.MustNewConstMetric(m.scheduledRecoveryRecovered, prometheus.CounterValue, float64(scheduled.RecoveryRecovered)) ch <- prometheus.MustNewConstMetric(m.scheduledExecutionStarted, prometheus.CounterValue, float64(scheduled.ExecutionStarted)) ch <- prometheus.MustNewConstMetric(m.scheduledExecutionCompleted, prometheus.CounterValue, float64(scheduled.ExecutionCompleted)) ch <- prometheus.MustNewConstMetric(m.scheduledExecutionFailed, prometheus.CounterValue, float64(scheduled.ExecutionFailed)) ch <- prometheus.MustNewConstMetric(m.scheduledCapacityDeferrals, prometheus.CounterValue, float64(scheduled.CapacityDeferrals)) ch <- prometheus.MustNewConstMetric(m.scheduledDiscoveryFailures, prometheus.CounterValue, float64(scheduled.DiscoveryFailures)) ch <- prometheus.MustNewConstMetric(m.scheduledPhaseDuration, prometheus.CounterValue, scheduled.DiscoveryDuration.Seconds(), "discovery") ch <- prometheus.MustNewConstMetric(m.scheduledPhaseDuration, prometheus.CounterValue, scheduled.RecoveryDuration.Seconds(), "recovery") ch <- prometheus.MustNewConstMetric(m.scheduledPhaseDuration, prometheus.CounterValue, scheduled.ExecutionDuration.Seconds(), "execution") } } func boolMetric(value bool) float64 { if value { return 1 } return 0 } func (m *metricsCollector) collectAuthority( ch chan<- prometheus.Metric, authority string, snapshotFunc func() authorityCheckSnapshot, ) { snapshot := authorityCheckSnapshot{} if snapshotFunc != nil { snapshot = snapshotFunc() } ch <- prometheus.MustNewConstMetric(m.authorityChecks, prometheus.CounterValue, float64(snapshot.Checks), authority) ch <- prometheus.MustNewConstMetric(m.authorityFailures, prometheus.CounterValue, float64(snapshot.Failures), authority) ch <- prometheus.MustNewConstMetric(m.authorityDuration, prometheus.CounterValue, snapshot.Duration.Seconds(), authority) ch <- prometheus.MustNewConstMetric(m.authorityLastSuccess, prometheus.GaugeValue, unixSeconds(snapshot.LastSuccess), authority) } func unixSeconds(value time.Time) float64 { if value.IsZero() { return 0 } return float64(value.UnixNano()) / float64(time.Second) }