""" Shared extraction logic for VictoriaMetrics pipelines (main + backfill). """ import hashlib import json import logging import time from datetime import datetime, timezone from typing import Iterator import requests logger = logging.getLogger(__name__) RETRY_ATTEMPTS = 3 RETRY_BACKOFF = 2 DATASET_NAME = "vm_metrics" def labels_hash(metric: dict) -> str: """Stable hash of metric labels (excluding __name__) for deduplication.""" labels = {k: v for k, v in sorted(metric.items()) if k != "__name__"} return hashlib.md5(json.dumps(labels, sort_keys=True).encode()).hexdigest() def fetch_query_range( base_url: str, query: str, start: int, end: int, step: int, ) -> list[dict]: """POST to /api/v1/query_range with retries. Returns the result array.""" url = base_url.rstrip("/") + "/query_range" params = {"query": query, "start": start, "end": end, "step": step} for attempt in range(1, RETRY_ATTEMPTS + 1): try: resp = requests.post(url, data=params, timeout=300) resp.raise_for_status() data = resp.json() if data.get("status") != "success": raise RuntimeError( f"query_range error: {data.get('errorType', '?')}: " f"{data.get('error', 'unknown')}" ) return data["data"]["result"] except requests.RequestException as e: if attempt == RETRY_ATTEMPTS: raise wait = RETRY_BACKOFF ** attempt logger.warning( "Attempt %d/%d failed for %s: %s. Retrying in %ds...", attempt, RETRY_ATTEMPTS, url, e, wait, ) time.sleep(wait) def parse_query_range_result( results: list[dict], kubernetes: str, provider: str, cloud_region: str, keep_labels: list[str] | None = None, ) -> Iterator[dict]: """Flatten query_range result into one dict per (series, timestamp) pair. If keep_labels is provided, only those metric labels are retained (strips extra labels that VictoriaMetrics may preserve on no-op aggregations). """ for series in results: raw_metric = series.get("metric", {}) if keep_labels is not None: metric = {k: v for k, v in raw_metric.items() if k in keep_labels} else: metric = {k: v for k, v in raw_metric.items() if k != "__name__"} lhash = labels_hash(metric) for ts, value_str in series.get("values", []): ts_int = int(ts) row = dict(metric) row["metric_timestamp"] = ts_int row["metric_value"] = float(value_str) row["metric_labels_hash"] = lhash row["metric_date"] = datetime.fromtimestamp(ts_int, tz=timezone.utc).date() row["kubernetes"] = kubernetes row["provider"] = provider row["cloud_region"] = cloud_region yield row