from pathlib import Path import pytest from pingcap_datapipelines.config import PipelineConfig from pingcap_datapipelines.extract import build_s3_client def s3_config(profile: str | None = None) -> PipelineConfig: s3 = { "region": "us-west-2", "credentials": { "access_key_id_env": "AWS_ACCESS_KEY_ID", "secret_access_key_env": "AWS_SECRET_ACCESS_KEY", "session_token_env": "AWS_SESSION_TOKEN", }, } if profile: s3["profile"] = profile return PipelineConfig( path=Path("source.yaml"), data={ "source": {"name": "example_snapshot", "type": "s3"}, "target": { "project": "gcp-digital-bi", "location": "us-west2", "datasets": {"raw": "raw", "final": "final", "control": "control"}, }, "gcs": { "bucket": "bucket", "snapshot_prefix": "snapshots", "manifest_prefix": "manifests", "runtime_log_prefix": "logs", }, "s3": s3, }, ) def test_build_s3_client_prefers_explicit_credentials(monkeypatch) -> None: calls = {} def fake_client(service, **kwargs): calls["service"] = service calls["kwargs"] = kwargs return "client" monkeypatch.setenv("AWS_ACCESS_KEY_ID", "key") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret") monkeypatch.setenv("AWS_SESSION_TOKEN", "token") monkeypatch.setattr("pingcap_datapipelines.extract.boto3.client", fake_client) assert build_s3_client(s3_config(profile="pingcap-datalake")) == "client" assert calls == { "service": "s3", "kwargs": { "region_name": "us-west-2", "aws_access_key_id": "key", "aws_secret_access_key": "secret", "aws_session_token": "token", }, } def test_build_s3_client_uses_profile_when_explicit_credentials_are_absent(monkeypatch) -> None: calls = {} class FakeSession: def __init__(self, *, profile_name, region_name): calls["profile_name"] = profile_name calls["region_name"] = region_name def client(self, service): calls["service"] = service return "profile-client" monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) monkeypatch.setattr("pingcap_datapipelines.extract.boto3.Session", FakeSession) assert build_s3_client(s3_config(profile="pingcap-datalake")) == "profile-client" assert calls == {"profile_name": "pingcap-datalake", "region_name": "us-west-2", "service": "s3"} def test_build_s3_client_requires_credentials_or_profile(monkeypatch) -> None: monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) with pytest.raises(RuntimeError, match="required S3 credentials are not set"): build_s3_client(s3_config())