from __future__ import annotations import argparse import json from pathlib import Path import sys REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT)) from ingestion.common.cloud_lake import client from ingestion.common.load_config import LoadConfigBase, load_config_file from ingestion.common.sql import split_sql def main() -> None: parser = argparse.ArgumentParser(description="Run idempotent warehouse setup from a load source config.") parser.add_argument("--source", required=True, help="Path to the load source.yaml.") args = parser.parse_args() # Loader-neutral: setup only needs the storage-agnostic base (name, DSN, setup files), # so it loads any load source — s3_bulk_load, gcs_bulk_load, or a future loader — the same way. config = load_config_file(resolve_repo_path(args.source), LoadConfigBase) executed = run_databend_setup(config) print(json.dumps({"source": config.name, "databend": {"executed": executed}}, indent=2)) def run_databend_setup(config: LoadConfigBase) -> list[str]: files = config.setup_files if not files: return [] cursor = client(config).cursor() executed = [] for file_name in files: path = resolve_repo_path(file_name) for statement in split_sql(path.read_text()): cursor.execute(statement) executed.append(str(path.relative_to(REPO_ROOT))) return executed def resolve_repo_path(file_name: str) -> Path: path = Path(file_name) if path.is_absolute(): return path return REPO_ROOT / path if __name__ == "__main__": main()