from __future__ import annotations import json import subprocess import sys from datetime import UTC, datetime from pathlib import Path from typing import Any from ingestion.common.aws import aws_env from .config import LoadConfig def write_event(config: LoadConfig, manifest: dict, event: dict[str, Any]) -> dict[str, str]: """Append one event to the local run log. The log is uploaded once at the end of the run by ``upload_runtime_log`` — not per event — so an N-table run does not re-send a steadily growing object N times (O(N^2) bytes).""" run_date = manifest["run_date"] run_id = manifest["run_id"] record = { "event_schema_version": 1, "source_name": manifest["source_name"], "cluster_id": manifest.get("cluster_id", ""), "run_date": run_date, "run_id": run_id, "export_task_id": manifest.get("export_task_id", ""), "created_at": datetime.now(UTC).isoformat(), **event, } output_path = runtime_log_path(config, run_date, run_id) output_path.parent.mkdir(parents=True, exist_ok=True) with output_path.open("a") as file: file.write(json.dumps(record, sort_keys=True) + "\n") return {"local_path": str(output_path), "s3_uri": manifest["runtime_log_uri"]} def upload_runtime_log(config: LoadConfig, manifest: dict) -> dict[str, str] | None: """Upload the complete run log to S3 once, after the run. Best-effort: a failed upload must not abort an otherwise-successful load — the raw data has already landed and the log remains on local disk. Returns the destination (with an ``uploaded`` flag), or None if there is nothing to upload. """ output_path = runtime_log_path(config, manifest["run_date"], manifest["run_id"]) if not output_path.exists(): return None s3_uri = manifest["runtime_log_uri"] try: subprocess.run( ["aws", "s3", "cp", str(output_path), s3_uri, "--only-show-errors"], check=True, env=aws_env(config), ) except (subprocess.CalledProcessError, OSError) as error: print( json.dumps({"runtime_log_upload_failed": s3_uri, "error": str(error)}), file=sys.stderr, ) return {"local_path": str(output_path), "s3_uri": s3_uri, "uploaded": "false"} return {"local_path": str(output_path), "s3_uri": s3_uri, "uploaded": "true"} def runtime_log_path(config: LoadConfig, run_date: str, run_id: str) -> Path: return config.root / "runs" / run_date / run_id / "events.jsonl"