"""Make referenced dbt source datasets requestable in the DAX inventory. The command is dry-run by default. Added rows are active requestable ownership metadata; existing active referenced source rows are promoted to requestable without changing their owner. Example from ``server/``:: .venv/bin/python scripts/sync_data_access_lineage_roots.py \ app/domains/data_access/lineage_catalogs/gcp-digital-bi-dbt.json \ ../deploy/data-access/datasets/gcp-digital-bi-dbt.json --commit """ from __future__ import annotations import argparse import json from pathlib import Path from app.domains.data_access.lineage_inventory import ( DEFAULT_LINEAGE_ROOT_OWNER_EMAIL, sync_lineage_root_datasets, ) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("catalog", type=Path) parser.add_argument("inventory", type=Path) parser.add_argument( "--default-owner-email", default=DEFAULT_LINEAGE_ROOT_OWNER_EMAIL, ) parser.add_argument("--commit", action="store_true") args = parser.parse_args() catalog = _load_object(args.catalog, label="lineage catalog") inventory = _load_object(args.inventory, label="dataset inventory") requestable_before = { (str(row["project_id"]), str(row["dataset_id"])): ( row.get("requestable") is True ) for row in inventory.get("datasets", []) if isinstance(row, dict) } projected, additions = sync_lineage_root_datasets( inventory, catalog, default_owner_email=args.default_owner_email, ) for addition in additions: print( "ADD " f"{addition.project_id}.{addition.dataset_id} " f"owner={addition.owner_email} requestable=true", ) for row in projected["datasets"]: locator = (str(row["project_id"]), str(row["dataset_id"])) if ( requestable_before.get(locator) is False and row.get("requestable") is True ): print(f"PROMOTE {locator[0]}.{locator[1]} requestable=true") prefix = "COMMIT" if args.commit else "DRY RUN" print(f"{prefix}: additions={len(additions)} total={len(projected['datasets'])}") if args.commit: args.inventory.write_text( json.dumps(projected, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) def _load_object(path: Path, *, label: str) -> dict: value = json.loads(path.read_text(encoding="utf-8")) if not isinstance(value, dict): raise ValueError(f"{label} must be a JSON object") return value if __name__ == "__main__": main()