from pathlib import Path from pingcap_datapipelines.config import PipelineConfig, load_config def test_materialize_final_is_false_without_final_dataset() -> None: config = PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "example_source", "type": "tidbcloud"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "control": "control"}, }, "load": {"materialize_final": False}, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) assert config.final_dataset is None assert config.materialize_final is False def test_materialize_final_defaults_to_true_when_final_dataset_exists() -> None: config = PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "nexuscrm", "type": "tidbcloud"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) assert config.final_dataset == "final" assert config.materialize_final is True def test_query_sql_path_and_final_table_name_are_configurable() -> None: config = PipelineConfig( path=Path("configs/sources/helix.yaml"), data={ "source": { "name": "helix", "type": "mysql_query", "query": {"sql_file": "../queries/helix/cloud_accounts.sql"}, }, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "helix_raw", "final": "helix", "control": "helix_control"}, }, "load": {"final_table_names": {"cloud_accounts": "cloud_accounts"}}, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) assert config.query_sql_path == Path("configs/queries/helix/cloud_accounts.sql").resolve() assert config.final_table_name_for("cloud_accounts") == "cloud_accounts" assert config.final_table_name_for("other") is None def test_helix_multi_query_tables_keep_contracts_with_each_sql_file() -> None: config = load_config(Path("configs/sources/helix.yaml")) assert [table["table"] for table in config.query_tables] == ["cloud_accounts", "users"] assert config.query_sql_path_for(config.query_tables[0]).name == "cloud_accounts.sql" assert config.query_sql_path_for(config.query_tables[1]).name == "users.sql" assert config.primary_key_for("cloud_accounts") == ["scope_id"] assert config.primary_key_for("users") == ["user_id"] assert config.final_table_name_for("users") == "pingcap_users"