"""Load committed dbt seed CSVs into Databend through the Rust driver. This is a manual fallback for local fixture loading. Normal dbt and Airflow execution should use `dbt seed`; the transform extra pins the tested adapter fork branch that supports gateway-safe seed INSERTs. All seed columns are created as VARCHAR; staging models cast as needed (e.g. amounts -> DOUBLE). Usage: uv run --env-file .env -- python transform/dbt/engine/load_seeds.py """ from __future__ import annotations import argparse import csv import os from pathlib import Path from databend_driver import BlockingDatabendClient REPO_ROOT = Path(__file__).resolve().parents[3] DBT_ROOT = REPO_ROOT / "transform" / "dbt" # The neutral seeds dir plus any source that ships its own seeds/. Discovered, not # enumerated, so onboarding a seed-bearing source needs no engine edit. SEED_DIRS = [DBT_ROOT / "seeds", *sorted((DBT_ROOT / "sources").glob("*/seeds"))] TARGET_DB = "transform_seeds" def _dsn() -> str: dsn = os.environ["DATABEND_DSN"] if dsn.startswith("lake://"): dsn = "databend://" + dsn[len("lake://") :] if "warehouse=" not in dsn: dsn += ("&" if "?" in dsn else "?") + f"warehouse={os.environ.get('DATABEND_WAREHOUSE', 'default')}" return dsn def _quote(v: str) -> str: return "NULL" if v == "" else "'" + v.replace("'", "''") + "'" def _seed_paths(seed_dirs: list[Path] = SEED_DIRS) -> dict[str, Path]: paths: dict[str, Path] = {} for seed_dir in seed_dirs: for csv_path in sorted(seed_dir.glob("*.csv")): name = csv_path.stem if name in paths: raise SystemExit(f"duplicate seed name {name!r}: {paths[name]} and {csv_path}") paths[name] = csv_path return paths def _selected_seed_paths(selected: list[str]) -> list[Path]: paths = _seed_paths() if not selected: return [paths[name] for name in sorted(paths)] missing = sorted(set(selected) - set(paths)) if missing: raise SystemExit(f"requested seeds not found: {', '.join(missing)}") return [paths[name] for name in sorted(set(selected))] def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Load committed dbt seed CSVs into Databend.") parser.add_argument( "--seed", action="append", default=[], help="Seed CSV stem to load. May be supplied more than once. Defaults to every discovered seed.", ) return parser.parse_args() def main() -> None: args = _parse_args() seed_paths = _selected_seed_paths(args.seed) cur = BlockingDatabendClient(_dsn()).cursor() cur.execute(f"CREATE DATABASE IF NOT EXISTS {TARGET_DB}") for csv_path in seed_paths: name = csv_path.stem with csv_path.open(newline="") as f: rows = list(csv.reader(f)) header, data = rows[0], rows[1:] cols = ", ".join(f"{c} VARCHAR" for c in header) cur.execute(f"CREATE OR REPLACE TABLE {TARGET_DB}.{name} ({cols})") if data: # chunk inserts to keep statements a reasonable size for i in range(0, len(data), 500): chunk = data[i : i + 500] values = ", ".join("(" + ", ".join(_quote(v) for v in r) + ")" for r in chunk) cur.execute(f"INSERT INTO {TARGET_DB}.{name} VALUES {values}") print(f"loaded {TARGET_DB}.{name}: {len(data)} rows, {len(header)} cols") if __name__ == "__main__": main()