"""Runtime-log upload is once-per-run and non-fatal (issue 09). write_event used to aws-s3-cp the entire growing events.jsonl after every table — O(N^2) bytes for an N-table run — and a failed cp (check=True) aborted an otherwise successful load. Now write_event only appends locally; upload_runtime_log sends the complete file once, best-effort. Runnable without a test harness:: .venv/bin/python tests/test_runtime_log_upload.py """ from __future__ import annotations import pathlib import subprocess import sys import tempfile from types import SimpleNamespace sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from ingestion.load.s3_bulk_load.engine import runtime_log def _config(root: pathlib.Path) -> SimpleNamespace: return SimpleNamespace(root=root, aws_profile="p", aws_region="us-west-2") def _manifest() -> dict: return { "run_date": "2026-05-26", "run_id": "run-123", "source_name": "nexuscrm", "runtime_log_uri": "s3://bucket/runs/2026-05-26/run-123/events.jsonl", } def test_write_event_appends_locally_without_uploading() -> None: calls: list = [] original = runtime_log.subprocess.run runtime_log.subprocess.run = lambda *a, **k: calls.append(a) or SimpleNamespace(returncode=0) with tempfile.TemporaryDirectory() as tmp: config = _config(pathlib.Path(tmp)) manifest = _manifest() try: runtime_log.write_event(config, manifest, {"event_type": "raw_snapshot_loaded", "status": "loaded"}) runtime_log.write_event(config, manifest, {"event_type": "raw_snapshot_loaded", "status": "loaded"}) finally: runtime_log.subprocess.run = original log_path = runtime_log.runtime_log_path(config, manifest["run_date"], manifest["run_id"]) lines = log_path.read_text().strip().splitlines() assert calls == [], "write_event must not upload per event" assert len(lines) == 2, "both events should be appended locally" def test_upload_runtime_log_sends_whole_file_once() -> None: calls: list = [] original = runtime_log.subprocess.run runtime_log.subprocess.run = lambda *a, **k: calls.append(a[0]) or SimpleNamespace(returncode=0) with tempfile.TemporaryDirectory() as tmp: config = _config(pathlib.Path(tmp)) manifest = _manifest() try: runtime_log.write_event(config, manifest, {"status": "loaded"}) runtime_log.write_event(config, manifest, {"status": "loaded"}) result = runtime_log.upload_runtime_log(config, manifest) finally: runtime_log.subprocess.run = original assert len(calls) == 1, f"expected exactly one upload, got {len(calls)}" assert calls[0][:3] == ["aws", "s3", "cp"] assert result == {"local_path": result["local_path"], "s3_uri": manifest["runtime_log_uri"], "uploaded": "true"} def test_upload_failure_is_non_fatal() -> None: def boom(*a, **k): raise subprocess.CalledProcessError(1, a[0]) original = runtime_log.subprocess.run runtime_log.subprocess.run = boom with tempfile.TemporaryDirectory() as tmp: config = _config(pathlib.Path(tmp)) manifest = _manifest() try: runtime_log.write_event(config, manifest, {"status": "loaded"}) result = runtime_log.upload_runtime_log(config, manifest) # must not raise finally: runtime_log.subprocess.run = original assert result is not None and result["uploaded"] == "false" def test_upload_noop_when_no_log() -> None: with tempfile.TemporaryDirectory() as tmp: result = runtime_log.upload_runtime_log(_config(pathlib.Path(tmp)), _manifest()) assert result is None if __name__ == "__main__": cases = [value for name, value in sorted(globals().items()) if name.startswith("test_") and callable(value)] for case in cases: case() print(f"ok {case.__name__}") print(f"\n{len(cases)} passed")