from __future__ import annotations from pathlib import Path from typing import Any import yaml from ingestion.common.contract_overrides import apply_raw_contract_overrides from ingestion.common.load_config import REPO_ROOT # Shared raw-snapshot *product* contract generation for every loader. The load stage # materializes only the raw layer and owns this product contract, inheriting schema, keys, # and classifications from the upstream `source` contract (ADR 0015/0016). Only the producer # ref differs per loader, so this lives in common rather than being duplicated. AUTHORED_SCHEMA_VERSION = "0.1.0" PRODUCT_VERSION = "0.1.0" def generate_raw_contracts(config, producer_ref: str) -> list[Path]: """Generate the raw-snapshot product contracts from this load source's upstream `source` contracts. ``producer_ref`` is the loader engine that materializes the raw tables.""" output_dir = config.root / "contracts" / "generated" output_dir.mkdir(parents=True, exist_ok=True) paths: list[Path] = [] for ref in config.source_contract_refs: src = yaml.safe_load((REPO_ROOT / ref).read_text()) table = src["source"]["identifiers"]["table"] paths.append(_write_yaml(output_dir / f"{config.name}.{table}.raw-snapshot.contract.yaml", raw_product(config, src, producer_ref))) return paths def raw_product(config, src: dict[str, Any], producer_ref: str) -> dict[str, Any]: identifiers = src["source"]["identifiers"] table = identifiers["table"] slug = table.lower() raw_db = config.cloud_lake_databases["raw"] domain = src["domain"]["name"] # TiDB sources identify the namespace as `database`, BigQuery `dataset`, external as `bucket`. namespace = identifiers.get("database") or identifiers.get("dataset") or identifiers.get("bucket") contract = { "schema_version": AUTHORED_SCHEMA_VERSION, "kind": "product", "id": f"{domain}.{config.name}.{slug}_snapshot", "name": f"{config.name} {table} raw snapshot", "version": PRODUCT_VERSION, "layer": "raw", "owner": src["owner"], "domain": src["domain"], "purpose": ( f"Source-faithful per-snapshot landing of {namespace}.{table} in the lakehouse raw layer." ), "usage": { "intended": ["Exploration", "Audit and provenance", "Derivation of staging-layer state"], "prohibited": [ "Authoritative curated reporting without a downstream curated contract.", "Intra-day event reconstruction.", ], }, "source": src["source"], "assets": [ { "id": f"{slug}_raw_snapshot", "kind": "table", "ref": f"{raw_db}.{config.name}_{slug}_snapshots", "materialization": "external", "produced_by": {"kind": "python", "ref": producer_ref}, } ], "schema": src["schema"], "freshness": src["freshness"], "tests": _placeholder_tests(slug, "raw"), "declared_lineage": {"upstreams": [{"kind": "source", "ref": src["id"]}]}, "governance": { "classifications": src["governance"]["classifications"], "access": {"default": "restricted"}, "retention": src["governance"]["retention"], "audit": {"required": True, "events": ["export", "agent_action"]}, }, "agent_permissions": {"default": "deny"}, "compatibility": {"change_policy": "semver"}, "extensions": { "primary_key": src["extensions"]["primary_key"], "watermark_columns": src["extensions"]["watermark_columns"], "row_estimate": src["extensions"].get("row_estimate"), "known_blind_spots": src["extensions"].get("known_blind_spots", []), }, } override_for = getattr(config, "raw_contract_override_for", None) overrides = override_for(table) if callable(override_for) else {} return apply_raw_contract_overrides(contract, overrides) def _placeholder_tests(table_slug: str, layer: str) -> list[dict[str, Any]]: return [ { "id": f"{table_slug}_{layer}_smoke", "type": "sql", "severity": "informational", "description": ( f"Placeholder informational smoke check for the {layer} layer of {table_slug}. " "Replace with concrete row-count, freshness, or key-uniqueness assertions." ), "sql": "select 1", } ] def _write_yaml(path: Path, data: dict[str, Any]) -> Path: path.write_text(yaml.safe_dump(data, sort_keys=False, allow_unicode=False)) return path