from __future__ import annotations import os from dataclasses import dataclass from pathlib import Path from typing import Any import yaml from ingestion.common.env import load_secrets, require_env @dataclass(frozen=True) class SourceConfig: path: Path data: dict[str, Any] @property def root(self) -> Path: return self.path.parent @property def source_name(self) -> str: return self.data["source"]["name"] @property def source_database(self) -> str: return self.data["source"]["database"] @property def source_dsn(self) -> str: return require_env(self.data["source"]["connection_env"]) @property def watermark_candidates(self) -> list[str]: return list(self.data["source"].get("watermark_candidates", [])) @property def selected_tables(self) -> list[str]: return list(self.data.get("tables", {}).get("include", [])) @property def excluded_tables(self) -> list[str]: return list(self.data.get("tables", {}).get("exclude", [])) @property def require_primary_key(self) -> bool: return bool(self.data.get("tables", {}).get("require_primary_key", False)) @property def tidb_cluster_id(self) -> str: return str(self.data["source"]["tidb_cloud"]["cluster_id"]) @property def classification_rules(self) -> dict[str, Any]: governance = self.data.get("governance", {}) return { "baseline": list(governance.get("classifications", [])), "patterns": dict(governance.get("classification_patterns", {})), "name_rules": list(governance.get("classification_name_rules", [])), } @property def data_bucket(self) -> str: return self.data["s3"]["data"]["bucket"] @property def snapshot_prefix(self) -> str: return self.data["s3"]["data"]["snapshot_prefix"].rstrip("/") @property def metadata_bucket(self) -> str: return self.data["s3"]["metadata"]["bucket"] @property def manifest_prefix(self) -> str: return self.data["s3"]["metadata"]["manifest_prefix"].rstrip("/") @property def runtime_log_prefix(self) -> str: return self.data["s3"]["metadata"]["runtime_log_prefix"].rstrip("/") @property def aws_region(self) -> str | None: return self.data.get("aws", {}).get("region") or os.environ.get("AWS_REGION") @property def aws_profile(self) -> str | None: return self.data.get("aws", {}).get("profile") or os.environ.get("AWS_PROFILE") @property def tidb_cli_profile(self) -> str | None: return self.data.get("tidb_cloud_cli_profile") or os.environ.get("TIDB_CLOUD_CLI_PROFILE") @property def tidb_cloud_credentials_env(self) -> dict[str, str]: return self.data["source"].get("tidb_cloud", {}).get("credentials", {}) @property def tidb_cloud_public_key_env(self) -> str: return self.tidb_cloud_credentials_env.get("public_key_env", "TIDB_CLOUD_PUBLIC_KEY") @property def tidb_cloud_private_key_env(self) -> str: return self.tidb_cloud_credentials_env.get("private_key_env", "TIDB_CLOUD_PRIVATE_KEY") @property def tidb_cloud_public_key(self) -> str | None: # API key the `ticloud` profile is bootstrapped from. Per-source (secrets.env wins # over repo-root .env) so the credential travels with the source, not the machine. return os.environ.get(self.tidb_cloud_public_key_env) @property def tidb_cloud_private_key(self) -> str | None: return os.environ.get(self.tidb_cloud_private_key_env) @property def parquet_compression(self) -> str: return self.data.get("export", {}).get("parquet_compression", "ZSTD") @property def export_credentials_env(self) -> dict[str, str]: return self.data.get("s3", {}).get("export_credentials", {}) @property def export_access_key_id_env(self) -> str: return self.export_credentials_env.get("access_key_id_env", "TIDB_EXPORT_AWS_ACCESS_KEY_ID") @property def export_secret_access_key_env(self) -> str: return self.export_credentials_env.get("secret_access_key_env", "TIDB_EXPORT_AWS_SECRET_ACCESS_KEY") @property def export_access_key_id(self) -> str: return require_env(self.export_access_key_id_env) @property def export_secret_access_key(self) -> str: return require_env(self.export_secret_access_key_env) @property def export_credential_env_names(self) -> list[str]: return [self.export_access_key_id_env, self.export_secret_access_key_env] def s3_run_uri(self, run_date: str, run_id: str) -> str: return ( f"s3://{self.data_bucket}/{self.snapshot_prefix}/" f"cluster={self.tidb_cluster_id}/run_date={run_date}/run_id={run_id}/" ) def manifest_s3_uri(self, run_date: str, run_id: str) -> str: return ( f"s3://{self.metadata_bucket}/{self.manifest_prefix}/" f"cluster={self.tidb_cluster_id}/run_date={run_date}/run_id={run_id}/manifest.json" ) def runtime_log_s3_uri(self, run_date: str, run_id: str) -> str: return ( f"s3://{self.metadata_bucket}/{self.runtime_log_prefix}/" f"cluster={self.tidb_cluster_id}/run_date={run_date}/run_id={run_id}/events.jsonl" ) def load_source_config(path: str | Path) -> SourceConfig: config_path = Path(path) data = yaml.safe_load(config_path.read_text()) secrets_file = data["source"].get("secrets_file", "secrets.env") load_secrets(config_path.parent / secrets_file) return SourceConfig(path=config_path, data=data)