#!/usr/bin/env python3 from __future__ import annotations import argparse import hashlib import json import os import re import stat from datetime import datetime, timezone from pathlib import Path from typing import Any from incremental_sync_state import atomic_write_json SHA256_RE = re.compile(r"^[0-9a-f]{64}$") SAFE_LABEL_RE = re.compile(r"^[A-Za-z0-9_.@/-]{1,128}$") SAFE_SUFFIX_RE = re.compile(r"^[A-Za-z0-9_.-]{1,80}$") UTC_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Recover one dead-letter replay entry only when its queue preimage and " "window match the current durable cursor." ) ) parser.add_argument("--replay-queue-file", required=True) parser.add_argument("--state-file", required=True) parser.add_argument("--entry-id", required=True) parser.add_argument("--expected-queue-sha256", required=True) parser.add_argument("--actor", required=True) parser.add_argument("--run-id", required=True) parser.add_argument("--output", required=True) return parser.parse_args() def load_object(path: Path) -> dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"{path} must contain a JSON object") return payload def sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def format_utc_now() -> str: return ( datetime.now(timezone.utc) .replace(microsecond=0) .isoformat() .replace("+00:00", "Z") ) def require_regular_file(path: Path, label: str) -> os.stat_result: if path.is_symlink(): raise ValueError(f"{label} must not be a symlink") try: file_stat = path.stat() except FileNotFoundError as exc: raise FileNotFoundError(f"{label} does not exist: {path}") from exc if not stat.S_ISREG(file_stat.st_mode): raise ValueError(f"{label} must be a regular file") return file_stat def restore_file_metadata(path: Path, source_stat: os.stat_result) -> None: os.chmod(path, stat.S_IMODE(source_stat.st_mode)) try: os.chown(path, source_stat.st_uid, source_stat.st_gid) except PermissionError: current_stat = path.stat() if ( current_stat.st_uid != source_stat.st_uid or current_stat.st_gid != source_stat.st_gid ): raise with path.open("rb") as handle: os.fsync(handle.fileno()) def fsync_directory(path: Path) -> None: directory_fd = os.open(path, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) def recover_replay_entry( *, queue_file: Path, state_file: Path, entry_id: str, expected_queue_sha256: str, recovered_at: str, actor: str, run_id: str, backup_suffix: str, ) -> dict[str, Any]: queue_file = Path(os.path.abspath(str(queue_file.expanduser()))) state_file = Path(os.path.abspath(str(state_file.expanduser()))) expected_sha = expected_queue_sha256.strip().lower() if not SHA256_RE.fullmatch(expected_sha): raise ValueError("expected_queue_sha256 must be 64 lowercase hex characters") if not SAFE_LABEL_RE.fullmatch(actor): raise ValueError("actor contains unsupported characters") if not run_id.isdigit(): raise ValueError("run_id must contain only digits") if not SAFE_SUFFIX_RE.fullmatch(backup_suffix): raise ValueError("backup_suffix contains unsupported characters") if not UTC_TIMESTAMP_RE.fullmatch(recovered_at): raise ValueError("recovered_at must be a whole-second UTC timestamp") queue_stat = require_regular_file(queue_file, "replay queue file") require_regular_file(state_file, "sync state file") queue_preimage = queue_file.read_bytes() actual_sha = sha256_bytes(queue_preimage) if actual_sha != expected_sha: raise ValueError( "replay queue compare-and-swap failed: expected SHA256 does not match" ) payload = json.loads(queue_preimage.decode("utf-8")) if not isinstance(payload, dict) or not isinstance(payload.get("entries"), list): raise ValueError("replay queue must contain an entries array") matching = [ entry for entry in payload["entries"] if isinstance(entry, dict) and str(entry.get("id", "")) == entry_id ] if len(matching) != 1: raise ValueError("replay queue must contain exactly one matching entry") entry = matching[0] window_start = str(entry.get("window_start", "")).strip() window_end = str(entry.get("window_end", "")).strip() if entry_id != f"{window_start}__{window_end}": raise ValueError("replay entry ID does not match its immutable window") if str(entry.get("status", "")) != "dead_letter": raise ValueError("only a dead_letter replay entry can be recovered") state = load_object(state_file) current_cursor = str(state.get("next_window_start", "")).strip() if not current_cursor or current_cursor != window_start: raise ValueError("replay entry is not at the current cursor") raw_lead_ids = entry.get("lead_ids") if not isinstance(raw_lead_ids, list): raise ValueError("replay entry lead_ids must be an array") lead_ids = [str(item).strip() for item in raw_lead_ids if str(item).strip()] if not lead_ids or len(lead_ids) != len(raw_lead_ids): raise ValueError("replay entry must contain non-empty pending Lead IDs") if len(lead_ids) != len(set(lead_ids)): raise ValueError("replay entry contains duplicate pending Lead IDs") if int(entry.get("candidate_count", -1)) != len(lead_ids): raise ValueError("replay entry candidate_count does not match pending Lead IDs") completed_ids = { str(item).strip() for item in entry.get("completed_lead_ids", []) if str(item).strip() } all_candidate_ids = { str(item).strip() for item in entry.get("all_candidate_ids", []) if str(item).strip() } if set(lead_ids) & completed_ids: raise ValueError("pending and completed Lead IDs must be disjoint") if not set(lead_ids).issubset(all_candidate_ids) or not completed_ids.issubset( all_candidate_ids ): raise ValueError( "replay entry lineage does not cover pending and completed IDs" ) if queue_file.read_bytes() != queue_preimage: raise ValueError( "replay queue changed during recovery; refusing to overwrite it" ) backup_path = queue_file.with_name( f"{queue_file.name}.bak-{backup_suffix}-{expected_sha[:12]}" ) with backup_path.open("xb") as backup: backup.write(queue_preimage) backup.flush() os.fsync(backup.fileno()) restore_file_metadata(backup_path, queue_stat) fsync_directory(queue_file.parent) previous_attempt_count = int(entry.get("attempt_count", 0) or 0) previous_no_progress_count = int(entry.get("consecutive_no_progress_count", 0) or 0) entry["status"] = "failed" entry["consecutive_no_progress_count"] = 0 entry["recovery_count"] = int(entry.get("recovery_count", 0) or 0) + 1 entry["last_recovery"] = { "recovered_at": recovered_at, "actor": actor, "run_id": run_id, "preimage_sha256": expected_sha, "previous_status": "dead_letter", "previous_attempt_count": previous_attempt_count, "previous_consecutive_no_progress_count": previous_no_progress_count, } entry["updated_at"] = recovered_at atomic_write_json(queue_file, payload) restore_file_metadata(queue_file, queue_stat) readback = load_object(queue_file) readback_matching = [ item for item in readback.get("entries", []) if isinstance(item, dict) and str(item.get("id", "")) == entry_id ] if len(readback_matching) != 1: raise RuntimeError("recovered replay entry readback failed") readback_entry = readback_matching[0] if ( str(readback_entry.get("status", "")) != "failed" or int(readback_entry.get("consecutive_no_progress_count", -1)) != 0 or [str(item) for item in readback_entry.get("lead_ids", [])] != lead_ids ): raise RuntimeError("recovered replay entry invariant readback failed") return { "recovery_version": 1, "status": "recovered", "entry_id": entry_id, "window_start": window_start, "window_end": window_end, "pending_candidate_count": len(lead_ids), "attempt_count_preserved": previous_attempt_count, "consecutive_no_progress_count_before": previous_no_progress_count, "consecutive_no_progress_count_after": 0, "queue_before_sha256": expected_sha, "queue_after_sha256": sha256_bytes(queue_file.read_bytes()), "backup_path": str(backup_path), "backup_sha256": sha256_bytes(backup_path.read_bytes()), "recovered_at": recovered_at, "actor": actor, "run_id": run_id, } def main() -> int: args = parse_args() recovered_at = format_utc_now() result = recover_replay_entry( queue_file=Path(args.replay_queue_file), state_file=Path(args.state_file), entry_id=args.entry_id.strip(), expected_queue_sha256=args.expected_queue_sha256, recovered_at=recovered_at, actor=args.actor.strip(), run_id=args.run_id.strip(), backup_suffix=f"run-{args.run_id.strip()}", ) output_path = Path(args.output).expanduser().resolve() atomic_write_json(output_path, result) print(f"[ok] replay_recovery_summary={output_path}") print(f"[ok] replay_queue_after_sha256={result['queue_after_sha256']}") return 0 if __name__ == "__main__": raise SystemExit(main())