from __future__ import annotations import os from dataclasses import dataclass from ingestion.common.env import require_env from ingestion.common.load_config import LoadConfigBase, TableTargets, load_config_file # Re-exported so existing importers (and tests) keep ``from ...config import TableTargets``. __all__ = ["LoadConfig", "TableTargets", "load_load_config"] @dataclass(frozen=True) class LoadConfig(LoadConfigBase): """S3 bulk-load source: the storage-agnostic base plus S3 read credentials. The base supplies the Cloud Lake DSN, target databases, setup files, and source- contract resolution; this subclass adds only how to read the S3 bulk extract. The physical column schema for COPY comes from the run manifest, not from here. """ @property def aws_region(self) -> str | None: return self.data.get("s3", {}).get("region") or os.environ.get("AWS_REGION") @property def aws_profile(self) -> str | None: return self.data.get("aws", {}).get("profile") or os.environ.get("AWS_PROFILE") @property def s3_credentials_env(self) -> dict[str, str]: return self.data.get("s3", {}).get("credentials", {}) @property def s3_access_key_id_env(self) -> str: return self.s3_credentials_env.get("access_key_id_env", "S3_ACCESS_KEY_ID") @property def s3_secret_access_key_env(self) -> str: return self.s3_credentials_env.get("secret_access_key_env", "S3_SECRET_ACCESS_KEY") @property def s3_access_key_id(self) -> str: return require_env(self.s3_access_key_id_env) @property def s3_secret_access_key(self) -> str: return require_env(self.s3_secret_access_key_env) @property def s3_session_token_env(self) -> str | None: """Optional: an env var holding a session token, for temporary/STS/SSO credentials.""" return self.s3_credentials_env.get("session_token_env") @property def s3_session_token(self) -> str | None: name = self.s3_session_token_env return os.environ.get(name) if name else None @property def s3_credential_env_names(self) -> list[str]: names = [self.s3_access_key_id_env, self.s3_secret_access_key_env] if self.s3_session_token_env: names.append(self.s3_session_token_env) return names def load_load_config(path) -> LoadConfig: return load_config_file(path, LoadConfig)