from __future__ import annotations from dataclasses import dataclass from datetime import datetime from google.cloud import storage @dataclass(frozen=True) class GcsUri: bucket: str name: str def parse_gs_uri(uri: str) -> GcsUri: if not uri.startswith("gs://"): raise ValueError(f"not a GCS URI: {uri}") bucket_and_name = uri.removeprefix("gs://") bucket, _, name = bucket_and_name.partition("/") if not bucket: raise ValueError(f"not a GCS URI: {uri}") return GcsUri(bucket=bucket, name=name) def list_objects(uri: str, *, client: storage.Client | None = None) -> list[dict]: parsed = parse_gs_uri(uri if uri.endswith("/") else uri + "/") storage_client = client or storage.Client() blobs = storage_client.list_blobs(parsed.bucket, prefix=parsed.name) return [ { "name": blob.name, "uri": f"gs://{parsed.bucket}/{blob.name}", "size": int(blob.size or 0), "updated": blob.updated.isoformat() if blob.updated else None, "updated_ms": int(blob.updated.timestamp() * 1000) if blob.updated else None, } for blob in blobs if not blob.name.endswith("/") ] def upload_text(uri: str, text: str, *, content_type: str = "application/json") -> None: parsed = parse_gs_uri(uri) client = storage.Client() bucket = client.bucket(parsed.bucket) blob = bucket.blob(parsed.name) blob.upload_from_string(text, content_type=content_type) def download_text(uri: str) -> str: parsed = parse_gs_uri(uri) client = storage.Client() return client.bucket(parsed.bucket).blob(parsed.name).download_as_text() def list_manifest_uris(prefix_uri: str) -> list[tuple[str, datetime | None]]: parsed = parse_gs_uri(prefix_uri if prefix_uri.endswith("/") else prefix_uri + "/") client = storage.Client() manifests = [] for blob in client.list_blobs(parsed.bucket, prefix=parsed.name): if blob.name.endswith("/manifest.json"): manifests.append((f"gs://{parsed.bucket}/{blob.name}", blob.updated)) return sorted(manifests, key=lambda item: (item[1] or datetime.min, item[0]))