from datetime import UTC, datetime from pathlib import Path from pingcap_datapipelines.bq_load import no_new_manifest_result, run_load from pingcap_datapipelines.config import PipelineConfig def load_config(*, max_age_hours: int | None = 30) -> PipelineConfig: load = {"full_snapshot_delete": True} if max_age_hours is not None: load["manifest_freshness_max_age_hours"] = max_age_hours return PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "nexuscrm", "type": "tidbcloud", "database": "nexuscrm"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, "load": load, }, ) def test_run_load_returns_structured_stale_status_when_no_manifest_is_unloaded(monkeypatch) -> None: config = load_config() manifest_uri = "gs://bucket/manifests/source=nexuscrm/run_date=2026-08-27/run_id=run-1/manifest.json" manifest = { "run_id": "run-1", "run_date": "2026-08-27", "created_at": "2026-08-27T01:10:00+00:00", } monkeypatch.setattr("pingcap_datapipelines.bq_load.bigquery.Client", lambda **_: object()) monkeypatch.setattr("pingcap_datapipelines.bq_load.ensure_control_tables", lambda *_: None) monkeypatch.setattr( "pingcap_datapipelines.bq_load.list_manifest_uris", lambda _: [(manifest_uri, None)], ) monkeypatch.setattr("pingcap_datapipelines.bq_load.read_manifest", lambda _: manifest) monkeypatch.setattr("pingcap_datapipelines.bq_load.is_run_loaded", lambda *_: True) [result] = run_load(config, all_unloaded=True) assert result["status"] == "no_new_manifest" assert result["reason"] == "no_unloaded_manifest" assert result["candidate_manifest_count"] == 0 assert result["processed_manifest_count"] == 0 assert result["freshness"]["signal"] == "latest_published_manifest_age" assert result["freshness"]["status"] == "stale" assert result["freshness"]["reason"] == "exceeds_max_age" assert result["freshness"]["latest_run_id"] == "run-1" assert result["freshness"]["max_age_seconds"] == 30 * 60 * 60 def test_no_new_manifest_status_is_unknown_when_nothing_was_published() -> None: result = no_new_manifest_result( load_config(), mode="latest", manifests=[], now=datetime(2026, 9, 2, tzinfo=UTC), ) assert result["status"] == "no_new_manifest" assert result["freshness"] == { "signal": "latest_published_manifest_age", "status": "unknown", "reason": "no_published_manifest", "max_age_seconds": 30 * 60 * 60, } def test_no_new_manifest_status_does_not_claim_freshness_without_threshold() -> None: manifest_uri = "gs://bucket/manifests/source=nexuscrm/run_date=2026-09-02/run_id=run-1/manifest.json" result = no_new_manifest_result( load_config(max_age_hours=None), mode="latest", manifests=[ ( manifest_uri, { "run_id": "run-1", "run_date": "2026-09-02", "created_at": "2026-09-02T01:00:00Z", }, ) ], now=datetime(2026, 9, 2, 2, tzinfo=UTC), ) assert result["freshness"]["status"] == "unconfigured" assert result["freshness"]["reason"] == "freshness_threshold_not_configured" assert result["freshness"]["age_seconds"] == 60 * 60