from __future__ import annotations import json import subprocess from datetime import datetime def list_objects(gcs_uri: str) -> list[dict]: """List objects under a GCS prefix as ``[{name, size, last_modified_ms}]`` via ``gcloud``. ``name`` is the object path relative to the bucket; ``last_modified_ms`` is epoch milliseconds (from the object's ``update_time``). ``gcloud storage objects list`` only recurses under a sub-prefix with the ``**`` recursive wildcard; a bare prefix matches an object named exactly that and returns nothing. Shared by the BigQuery extract (manifest) and the GCS load (external discovery).""" prefix = gcs_uri if gcs_uri.endswith("/") else gcs_uri + "/" completed = subprocess.run( ["gcloud", "storage", "objects", "list", f"{prefix}**", "--format=json"], check=True, capture_output=True, text=True, ) items = json.loads(completed.stdout or "[]") return [ {"name": item["name"], "size": int(item.get("size", 0)), "last_modified_ms": _epoch_ms(item.get("update_time"))} for item in items ] def _epoch_ms(value: str | None) -> int | None: if not value: return None text = value.replace("Z", "+00:00") try: parsed = datetime.fromisoformat(text) except ValueError: parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z") return int(parsed.timestamp() * 1000)