import json import sys from hashlib import sha256 from pathlib import Path import pytest from pingcap_datapipelines.cli import main, parse_tables from pingcap_datapipelines.config import PipelineConfig from pingcap_datapipelines.models import Column, Table def test_parse_tables_rejects_an_explicit_empty_selection() -> None: with pytest.raises(ValueError, match="at least one exact table"): parse_tables(", ,") def test_inspect_compact_prints_one_complete_json_line(monkeypatch, capsys) -> None: config = PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "nexuscrm", "type": "tidbcloud", "database": "nexuscrm"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) tables = [ Table( database="nexuscrm", name="Account", rows_estimate=1, columns=[Column("Id", "varchar", "VARCHAR(18)", False, 1)], primary_key=["Id"], ), Table( database="nexuscrm", name="_System_OutboxEvent", rows_estimate=2, columns=[Column("Id", "varchar", "VARCHAR(18)", False, 1)], primary_key=["Id"], ), ] monkeypatch.setattr("pingcap_datapipelines.cli.load_config", lambda _: config) monkeypatch.setattr("pingcap_datapipelines.cli.inspect_tables", lambda _: tables) monkeypatch.setattr( sys, "argv", ["pingcap-pipeline", "inspect", "--source", "source.yaml", "--compact"], ) main() output = capsys.readouterr().out assert output.count("\n") == 1 assert json.loads(output) == { "source": "nexuscrm", "source_database": "nexuscrm", "table_count": 2, "selection": { "mode": "configured", "requested_tables": None, "resolved_tables": ["Account", "_System_OutboxEvent"], "qualified_patterns": ["nexuscrm.Account", "nexuscrm._System_OutboxEvent"], "qualified_patterns_sha256": sha256( b"nexuscrm.Account\nnexuscrm._System_OutboxEvent" ).hexdigest(), "canonicalization": "utf8_newline_joined_sorted_qualified_patterns", }, "tables": [ {"name": "Account", "rows_estimate": 1, "primary_key": ["Id"], "watermark_columns": []}, { "name": "_System_OutboxEvent", "rows_estimate": 2, "primary_key": ["Id"], "watermark_columns": [], }, ], } def test_inspect_explicit_tidb_subset_is_validated_and_hashed(monkeypatch, capsys) -> None: config = PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "nexuscrm", "type": "tidbcloud", "database": "nexuscrm"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) selected = [ Table("nexuscrm", "Account", 3, [], ["Id"]), Table("nexuscrm", "Order", 5, [], ["Id"]), ] calls: list[list[str]] = [] monkeypatch.setattr("pingcap_datapipelines.cli.load_config", lambda _: config) monkeypatch.setattr( "pingcap_datapipelines.cli.inspect_exact_tidb_tables", lambda _config, tables: calls.append(tables) or selected, ) monkeypatch.setattr( sys, "argv", [ "pingcap-pipeline", "inspect", "--source", "source.yaml", "--tables", "Order,Account", "--compact", ], ) main() payload = json.loads(capsys.readouterr().out) assert calls == [["Order", "Account"]] assert payload["table_count"] == 2 assert payload["selection"]["mode"] == "explicit_subset" assert payload["selection"]["requested_tables"] == ["Order", "Account"] assert payload["selection"]["resolved_tables"] == ["Account", "Order"] assert payload["selection"]["qualified_patterns"] == ["nexuscrm.Account", "nexuscrm.Order"] def test_load_passes_operator_bound_expected_tables(monkeypatch, capsys) -> None: config = PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "nexuscrm", "type": "tidbcloud", "database": "nexuscrm"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, }, ) calls: list[dict[str, object]] = [] monkeypatch.setattr("pingcap_datapipelines.cli.load_config", lambda _: config) monkeypatch.setattr( "pingcap_datapipelines.cli.run_load", lambda _config, **kwargs: calls.append(kwargs) or [{"status": "loaded"}], ) monkeypatch.setattr( sys, "argv", [ "pingcap-pipeline", "load", "--source", "source.yaml", "--manifest", "gs://bucket/manifest.json", "--tables", "Account,Order", ], ) main() assert json.loads(capsys.readouterr().out) == [{"status": "loaded"}] assert calls == [ { "manifest_uri": "gs://bucket/manifest.json", "latest": False, "all_unloaded": False, "tables": ["Account", "Order"], } ]