from __future__ import annotations import argparse import json from ingestion.common.cloud_lake import client from .infer import committed_path, ensure_stage, infer_columns, source_contract, write_contract from .load import external_load_run, sample_object from .storage import StorageAdapter from .strategy import build_strategy # Shared CLI orchestration for every object-store bulk loader. The dispatch (infer-source, # generate-contracts, manifest- vs external-mode load-run) is storage-agnostic; the only # loader-specific parts are which StorageAdapter to build, the manifest-mode ``load_run``, # and the source-config class. Each loader's ``cli.py`` is therefore pure wiring — keeping # this logic in one place rather than duplicated per loader (config-driven clean structure # principle; ADR 0017). def infer_source(config, storage: StorageAdapter): """Infer the external bucket's schema and (re)write the committed source contract. This is the deliberate bootstrap/promote action (a human runs it); the automatic load path never overwrites the committed contract, it routes drift to ``contracts/proposed/`` (ADR 0018).""" strategy = build_strategy(config.incremental) cursor = client(config).cursor() ensure_stage(cursor, storage, config) objects = storage.list_objects(config.external_uri) columns = infer_columns(cursor, config.stage_name, sample_object(strategy.increments(objects))) return write_contract(committed_path(config), source_contract(config, columns)) def run_external_load(config, storage: StorageAdapter, reload_key: int | None = None) -> list[dict]: return external_load_run(config, storage, build_strategy(config.incremental), reload_key=reload_key) def loader_main(*, description, storage_cls, load_run, generate_contracts, load_load_config) -> None: """Run a loader CLI. ``storage_cls`` is the loader's StorageAdapter (built from the config), ``load_run`` its manifest-driven loader, ``generate_contracts`` its raw-product generator, and ``load_load_config`` its source-config parser — the only loader-specific inputs.""" parser = argparse.ArgumentParser(description=description) parser.add_argument("--source", required=True, help="Path to the load source.yaml") subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("infer-source") # external sources: infer + solidify the source contract subparsers.add_parser("generate-contracts") load_parser = subparsers.add_parser("load-run") load_parser.add_argument("--manifest", help="Run manifest (manifest-driven sources only)") load_parser.add_argument("--tables", help="Comma-separated table names (manifest-driven sources)") load_parser.add_argument("--reload-key", type=int, help="Force (re)load this cursor value (external sources)") args = parser.parse_args() config = load_load_config(args.source) if args.command == "infer-source": print(json.dumps({"source_contract": str(infer_source(config, storage_cls(config)))}, indent=2)) elif args.command == "generate-contracts": paths = generate_contracts(config) print(json.dumps({"contracts": [str(path) for path in paths]}, indent=2)) elif args.command == "load-run": if config.is_external: loads = run_external_load(config, storage_cls(config), reload_key=args.reload_key) print(json.dumps({"loads": loads}, indent=2)) else: tables = [item.strip() for item in args.tables.split(",")] if args.tables else None print(json.dumps({"loads": load_run(config, args.manifest, tables)}, indent=2))