#!/usr/bin/env python3
"""Shared D3 engine consumed by environment adapters and standalone installers.

Test1/Test3, Staging and Prod install their environment adapters before calling
main(). Standalone installers reuse the Catalog helpers. Historical state formats remain storage contracts. Test2 deployment uses its
standalone workflows; this module is not an operator entrypoint.
"""

from __future__ import annotations

import argparse
import base64
import binascii
import copy
import hashlib
import json
import os
import re
import secrets
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Mapping, Sequence


SCRIPT_DIR = Path(__file__).resolve().parent
REPOSITORY_ROOT = SCRIPT_DIR.parent
RUNTIME_DIR = REPOSITORY_ROOT / "deployments" / "offline" / "runtime"
for import_root in (REPOSITORY_ROOT, SCRIPT_DIR, RUNTIME_DIR):
    if str(import_root) not in sys.path:
        sys.path.insert(0, str(import_root))

from common import (  # noqa: E402
    DeployError,
    activate_nginx_conf,
    align_runtime_admin_password,
    bool_env,
    db_exec,
    db_query_scalar,
    ensure_tidb_database,
    exp_hubspot_options_adapter_port,
    load_installed_pack_convergence_state,
    load_env_file,
    publish_symlink,
    RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_PLAN_VERSION,
    RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_TABLES,
    purge_reset_snapshot_connector_queue,
    require_commands,
    render_template,
    reset_drained_workflow_action_syncstate_for_packs,
    reset_snapshot_workflow_action_syncstate,
    resolve_frontend_dist_target,
    run_backend_setup,
    run_catalog_official_source_set,
    run_catalog_mutable_source_preconvergence,
    run_catalog_offline_candidate,
    runtime_deploy_pack_entries,
    runtime_pack_entry_name_and_source,
    runtime_pack_names,
    tidb_table_exists,
    tidb_branch_parent_timestamp,
    tidb_cloud_api,
    verify_frontend_dist,
    wait_for_backend_health,
    wait_for_tidb_branch_active,
)
import deploy_staging_env as staging  # noqa: E402
import test_slot_runtime as test_deploy  # noqa: E402
import deploy_offline_env as offline  # noqa: E402
from scripts.salesforce_migration.file_history_contract import (  # noqa: E402
    salesforce_history_archive_authority,
)
from deployment_runtime_state import runtime_paths, update_env_file  # noqa: E402
from validate_serviceification_bom import (  # noqa: E402
    D3_WORKER_ROLE_SHAPES,
    LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION,
    LOCAL_TEST3_RELEASE_CLASS,
    RESOLVED_SCHEMA_VERSION_V10,
    ValidationFailure,
    canonical_json_sha256,
    desired_v10_projection_from_resolved,
    deployment_bom_view as operational_bom,
    historical_deployment_bom_view,
    load_bom,
    parse_bom,
    validate_deployment_bom_view,
    verify_app_packs_artifacts,
)
from render_serviceification_test_stack import (  # noqa: E402
    D3_INTEGRATION_BINDINGS_SCHEMA,
    D3_RUNTIME_ATTEMPT_BUDGET,
    host_compatibility,
    render_worker_activation_inputs,
    render_worker_role_inputs,
    validate_app_engine_compatibility,
    validate_catalog_output_for_deployment,
)
from serviceification_deployment_tools import (  # noqa: E402
    FILES as DEPLOYMENT_TOOL_FILES,
    SCHEMA as DEPLOYMENT_TOOL_SCHEMA,
    ToolError as DeploymentToolError,
    require_setup_commands,
    verify_tools as verify_deployment_tools,
)

from scripts.lib.final_pack_contract import (  # noqa: E402
    FinalPackContractError,
    metadata_tree_sha256,
)
from scripts.bind_programmatic_oauth_client import (  # noqa: E402
    validate_client as validate_programmatic_oauth_client,
)


STATE_SCHEMA = "nexuscrm.serviceification-d3-test2-state/v1"
EVIDENCE_SCHEMA = "nexuscrm.serviceification-d3-test2-deployment-evidence/v1"
D3_APP_SHELL_RECEIPT_SCHEMA = "nexuscrm.serviceification-d3-app-shell/v2"
D3_APP_SHELL_RECEIPT_KEYS = frozenset({
    "schema", "status", "l2Commit", "configSha256", "appId",
    "frontendAppId", "isDefault", "navigationItemIds", "navigationRevision",
    "activeProfileCount", "defaultProfileCount", "mode",
})
D3_RESET_SNAPSHOT_CONNECTOR_QUEUE_RECEIPT_SCHEMA_VERSION = 1
TEST2_NGINX_CONF = Path("/etc/nginx/conf.d/00-nexuscrm-test.conf")
D3_NGINX_MARKER = "# NexusCRM D3 direct ingress"
CONTAINER_LABEL_KEY = "io.nexuscrm.serviceification-d3"
CONTAINER_LABEL_VALUE = "test2"
RUN_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")
SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}")
SUPPORTED_WORKER_FAMILIES = tuple(shape[0] for shape in D3_WORKER_ROLE_SHAPES)
PRE_PERMISSION_WORKER_FAMILIES = tuple(
    family for family in SUPPORTED_WORKER_FAMILIES
    if family != "runtime.permission-authoring-command-lifecycle.v1"
)
LEGACY_WORKER_FAMILIES = tuple(
    family for family in PRE_PERMISSION_WORKER_FAMILIES
    if family != "runtime.bulk-ingest.v1"
)
# Historical seals remain readable for existing deployments and recovery.
# Test2 Permission cutover requires the complete set in its installer.
READABLE_WORKER_FAMILY_SETS = tuple(map(frozenset, (
    LEGACY_WORKER_FAMILIES, PRE_PERMISSION_WORKER_FAMILIES, SUPPORTED_WORKER_FAMILIES,
)))


def worker_stem(work_family: str) -> str:
    if work_family not in SUPPORTED_WORKER_FAMILIES:
        raise DeployError(f"unknown Runtime Worker family: {work_family}")
    return work_family.removeprefix("runtime.").removesuffix(".v1").replace(".", "-")


LEGACY_WORKER_STEMS = tuple(worker_stem(family) for family in LEGACY_WORKER_FAMILIES)
SUPPORTED_WORKER_STEMS = tuple(worker_stem(family) for family in SUPPORTED_WORKER_FAMILIES)
READABLE_WORKER_STEM_SETS = tuple(
    frozenset(worker_stem(family) for family in families)
    for families in READABLE_WORKER_FAMILY_SETS
)
# Retained for read-only operational tooling that has not loaded a BOM. The
# deployment runner itself always uses the role set sealed by its BOM/state.
WORKER_STEMS = LEGACY_WORKER_STEMS
WORKER_FAMILIES = set(LEGACY_WORKER_FAMILIES)
OBJECT_STORAGE_WORKER_STEMS = frozenset({"bulk-ingest", "file-upload-lifecycle"})
D3_FILESYSTEM_STORAGE_DIRECTORY = "serviceification-d3-object-storage-v1"
D3_FILESYSTEM_STORAGE_MARKER = ".nexuscrm-test-storage-authority.json"
D3_FILESYSTEM_STORAGE_MARKER_SCHEMA = (
    "nexuscrm.serviceification-d3-test-storage-authority/v1"
)
L0_WORKER_RUNTIME_CONFIG_SCHEMA = "nexuscrm.serviceification-l0-worker-runtime-config/v1"
L0_WORKER_IMAGE_TRANSITION_SCHEMA = "nexuscrm.serviceification-l0-worker-image-transition/v1"
L0_SERVICE_IMAGE_TRANSITION_SCHEMA = "nexuscrm.serviceification-l0-service-image-transition/v1"
L0_SERVICE_IMAGE_FAILURE_EVIDENCE_SCHEMA = (
    "nexuscrm.serviceification-l0-service-image-failure-evidence/v1"
)
L0_SERVING_TLS_ROTATION_SCHEMA = "nexuscrm.serviceification-l0-serving-tls-rotation/v1"
D3_SERVING_TLS_VALIDITY_DAYS = 365
L0_PARALLELISM_MUTABLE_WORKER_ROLES = {"record-event-dispatch"}
L0_POLL_INTERVAL_MUTABLE_WORKER_ROLES = {"record-command-lifecycle"}
# A retained production-clone database can take several minutes to rebuild the
# process-local Runtime readiness projections after an image-only restart.  The
# serving transaction remains stopped and fail-closed during this interval; do
# not misclassify a healthy cold start as a failed deployment and abandon the
# resumable transaction.  This bound is only for container readiness and does
# not expand any Runtime request deadline.
D3_SERVING_READINESS_TIMEOUT_SECONDS = 480
D3_DOCKER_HEALTH_TIMEOUT_SECONDS = 30
CONNECTED_MAIL_CLIENT_ID_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_OAUTH_CLIENT_ID"
CONNECTED_MAIL_CLIENT_SECRET_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_OAUTH_CLIENT_SECRET"
CONNECTED_MAIL_DEFINITION_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_DEFINITION_FILE"
CONNECTED_MAIL_CALLBACK_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_CALLBACK_URI"
CONNECTED_MAIL_RETURN_BASE_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_RETURN_BASE_URL"
CONNECTED_MAIL_MASTER_KEY_ENV = "NEXUSCRM_INTEGRATION_CONNECTED_MAIL_MASTER_KEY"
CONNECTED_MAIL_PROVIDER_PATH = (
    "gmail-connected-mail-connector/integrations/connected_mail_gmail.json"
)
CONNECTED_MAIL_CONTAINER_PATH = "/run/nexuscrm/connected-mail/provider.json"
CONNECTED_MAIL_PUBLIC_PATH = "/api/connected-mailboxes/gmail"
RESET_SNAPSHOT_SCHEDULED_RUN_HYGIENE_REASON = (
    "test2 production-clone cutover: inherited legacy scheduled occurrence is not eligible to execute"
)


def reset_snapshot_connector_queue_purge_summary(
    purged: Mapping[str, int],
) -> dict[str, object]:
    """Record the exact reset-only connector sanitation contract in D3 state."""
    return {
        "connectorQueueSanitizationReceiptSchema": (
            D3_RESET_SNAPSHOT_CONNECTOR_QUEUE_RECEIPT_SCHEMA_VERSION
        ),
        "connectorEventsPurged": purged["connector_events"],
        "connectorWorkItemsPurged": purged["connector_work_items"],
        "postSanitizationConnectorEvents": 0,
        "postSanitizationConnectorWorkItems": 0,
        "connectorQueueSanitizationMode": "truncate",
        "connectorQueueSanitizationPlanVersion": (
            RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_PLAN_VERSION
        ),
        "connectorQueueSanitizationTables": [
            table_name for table_name, _ in RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_TABLES
        ],
    }


def require_reset_snapshot_connector_queue_purge_receipt(
    state: Mapping[str, Any],
) -> None:
    """Reject a D3 continuation whose reset receipt predates queue sanitation.

    D3 state schema v1 predates this repair. Keeping its envelope avoids
    stranding a stopped test2 transition, but a v1 baseline-reset receipt
    without this explicit sub-receipt may never move the deployment forward.
    Recovery/rollback remains able to restore the prior owner; a fresh D3 run
    creates a complete receipt before any new Catalog or Runtime mutation.
    """
    baseline_reset = state.get("baselineReset")
    if baseline_reset is None:
        return
    if not isinstance(baseline_reset, Mapping):
        raise DeployError("D3 baseline reset receipt is malformed")
    source = baseline_reset.get("source")
    if source == "retained-test2-database":
        # Retaining the current Test2 database is not a provider reset. Its
        # durable connector queue must remain intact, so the reset-only purge
        # receipt does not apply.
        return
    if source not in {
        "prod-tidb-reset-snapshot",
        "prod-tidb-reset-snapshot-recovered",
    }:
        raise DeployError("D3 baseline reset receipt has an unknown source")
    expected_tables = [
        table_name for table_name, _ in RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_TABLES
    ]
    if (
        baseline_reset.get("connectorQueueSanitizationReceiptSchema")
        != D3_RESET_SNAPSHOT_CONNECTOR_QUEUE_RECEIPT_SCHEMA_VERSION
        or baseline_reset.get("connectorQueueSanitizationMode") != "truncate"
        or baseline_reset.get("connectorQueueSanitizationPlanVersion")
        != RESET_SNAPSHOT_CONNECTOR_QUEUE_PURGE_PLAN_VERSION
        or baseline_reset.get("connectorQueueSanitizationTables") != expected_tables
    ):
        raise DeployError(
            "D3 baseline reset receipt predates the versioned connector queue purge; "
            "refusing forward resume. Recover or roll back the active transition, "
            "then start a fresh D3 run"
        )
    for key in (
        "connectorEventsPurged",
        "connectorWorkItemsPurged",
        "postSanitizationConnectorEvents",
        "postSanitizationConnectorWorkItems",
    ):
        value = baseline_reset.get(key)
        if not isinstance(value, int) or isinstance(value, bool) or value < 0:
            raise DeployError(f"D3 baseline reset has invalid connector sanitation count: {key}")
    if (
        baseline_reset["postSanitizationConnectorEvents"] != 0
        or baseline_reset["postSanitizationConnectorWorkItems"] != 0
    ):
        raise DeployError("D3 baseline reset connector sanitation receipt is not empty")


def new_uuid7() -> str:
    """Mint one canonical UUIDv7 without depending on the host Python version."""
    timestamp_ms = time.time_ns() // 1_000_000
    if timestamp_ms >= 1 << 48:
        raise DeployError("current time exceeds UUIDv7 timestamp range")
    raw = bytearray(timestamp_ms.to_bytes(6, "big") + secrets.token_bytes(10))
    raw[6] = (raw[6] & 0x0F) | 0x70
    raw[8] = (raw[8] & 0x3F) | 0x80
    return str(uuid.UUID(bytes=bytes(raw)))


BASELINE_RESET_ELIGIBLE_PHASES = {
    "artifacts-ready",
    "quiesce-in-progress",
    "quiesced",
    "runtime-apply-in-progress",
    "baseline-reset-in-progress",
}

CATALOG_AUTHORITY_TABLES = {
    "_System_CatalogAppInstallation",
    "_System_CatalogOrgState",
    "_System_CatalogPublication",
    "_System_CatalogPublicationActive",
}
RUNTIME_WORK_OWNERSHIP_TABLES = {"_System_RuntimeWorkOwnership"}


def required_table_sql(table_names: set[str]) -> str:
    values = ",".join(f"'{name}'" for name in sorted(table_names))
    return (
        "SELECT TABLE_NAME FROM information_schema.tables "
        f"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN ({values}) "
        "ORDER BY TABLE_NAME;"
    )


CATALOG_AUTHORITY_TABLES_SQL = required_table_sql(CATALOG_AUTHORITY_TABLES)
RUNTIME_WORK_OWNERSHIP_TABLES_SQL = required_table_sql(RUNTIME_WORK_OWNERSHIP_TABLES)

CATALOG_AUTHORITY_SQL = """
SELECT CONCAT_WS('\t',
  HEX(i.AppInstallationId), HEX(i.OrgId), i.ActiveReleaseDigest,
  CAST(i.CatalogRevision AS CHAR), HEX(s.RuntimeAudience),
  CAST(s.CatalogRevision AS CHAR), CAST(s.LastSecurityEpoch AS CHAR),
  a.ActivePublicationDigest, CAST(p.SchemaGeneration AS CHAR),
  CAST(p.EngineGeneration AS CHAR))
FROM _System_CatalogAppInstallation i
JOIN _System_CatalogOrgState s ON s.OrgId = i.OrgId
JOIN _System_CatalogPublicationActive a ON a.OrgId = i.OrgId
JOIN _System_CatalogPublication p
  ON p.OrgId = i.OrgId AND p.PublicationDigest = a.ActivePublicationDigest
WHERE i.AppId = 'io.nexuscrm.sfcrm'
  AND i.State = 'ACTIVE'
  AND i.ActiveReleaseDigest IS NOT NULL
  AND i.ActiveOperationId IS NULL
  AND s.ActiveOperationId IS NULL
  AND a.ActivePublicationPresent = 1
  AND a.ActivePublicationDigest IS NOT NULL
  AND p.CatalogRevision = s.CatalogRevision
  AND p.SecurityEpoch = s.LastSecurityEpoch;
""".strip()

CATALOG_PUBLISHER_TRUST_SQL = """
SELECT CONCAT_WS('\t', HEX(p.PublisherId), CAST(p.OwnershipEpoch AS CHAR), p.State,
  HEX(k.KeyId), HEX(k.Algorithm), HEX(k.PublicKey),
  DATE_FORMAT(k.ValidFrom, '%Y-%m-%dT%H:%i:%s.%fZ'),
  DATE_FORMAT(k.ValidUntil, '%Y-%m-%dT%H:%i:%s.%fZ'))
FROM _System_CatalogPublisher p
JOIN _System_CatalogPublisherKey k ON k.PublisherId = p.PublisherId
WHERE p.PublisherId = 'io.nexuscrm' AND k.State = 'ACTIVE'
ORDER BY k.KeyId;
""".strip()

CATALOG_PUBLISHER_TRANSITION_RECEIPT_SQL = """
SELECT CONCAT_WS('\t', HEX(OperationId), HEX(PublisherId),
  CAST(ExpectedOwnershipEpoch AS CHAR), CAST(ResultOwnershipEpoch AS CHAR),
  PreviousTrustDigest, ActiveTrustDigest)
FROM _System_CatalogPublisherTrustTransition
WHERE OperationId = {operation_id};
""".strip()

WORKER_OWNERSHIP_SQL = """
SELECT CONCAT_WS('\t', HEX(WorkFamily), HEX(DeploymentId), WorkerReleaseDigest,
  CAST(OwnershipEpoch AS CHAR), CAST(IsActive AS CHAR), HEX(OperationId))
FROM _System_RuntimeWorkOwnership
WHERE WorkFamily IN (
  'runtime.audit-history-fanout.v1',
  'runtime.bulk-ingest.v1',
  'runtime.file-upload-lifecycle.v1',
  'runtime.permission-authoring-command-lifecycle.v1',
  'runtime.platform-command.v1',
  'runtime.record-command-lifecycle.v1',
  'runtime.record-event-dispatch.v1',
  'runtime.rollup-delta.v1',
  'runtime.scheduled-flow.v1')
ORDER BY WorkFamily;
""".strip()


def canonical_bytes(value: Any) -> bytes:
    return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()


def sha256_bytes(value: bytes) -> str:
    return "sha256:" + hashlib.sha256(value).hexdigest()


def require_port(value: object, label: str) -> int:
    text = str(value).strip()
    if not text.isascii() or not text.isdecimal() or not 1 <= int(text) <= 65535:
        raise DeployError(f"{label} must be a port between 1 and 65535")
    return int(text)


@dataclass(frozen=True)
class D3Ports:
    sfcrm_http: int
    integration_http: int
    runtime_grpc: int
    runtime_admin: int
    worker_admin: Mapping[str, int]
    mcp_http: int | None = None

    def all_ports(self) -> dict[str, int]:
        return {
            "sfcrm_http": self.sfcrm_http,
            **({"mcp_http": self.mcp_http} if self.mcp_http is not None else {}),
            "integration_http": self.integration_http,
            "runtime_grpc": self.runtime_grpc,
            "runtime_admin": self.runtime_admin,
            **{f"worker_{stem}": port for stem, port in self.worker_admin.items()},
        }


def ports_with_mcp(ports: D3Ports, images: Mapping[str, Any]) -> D3Ports:
    if "mcp_worker" not in images:
        return ports
    port = require_port(ports.runtime_admin + 1, "MCP private HTTP port")
    if port in {v for k, v in ports.all_ports().items() if k != "mcp_http"}:
        raise DeployError("MCP private HTTP port collides with another selected role")
    return replace(ports, mcp_http=port)


def workflow_action_adapter_replay_url(ports: D3Ports) -> str:
    """Send adapter replays to Integration, independently of SFCRM OAuth/SOQL."""
    port = require_port(ports.integration_http, "D3 Integration replay port")
    return (
        f"http://127.0.0.1:{port}"
        "/webhooks/hubspot-crm-connector/workflow-action"
    )


def verify_workflow_action_adapter_options(port: int, options_token: str) -> None:
    """Exercise the adapter's OAuth and SOQL path after SFCRM becomes ready."""
    request = urllib.request.Request(
        f"http://127.0.0.1:{port}/hubspot/workflow-actions/options/campaigns",
        data=json.dumps({"options_token": options_token}).encode(),
        method="POST",
        headers={"Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(request, timeout=40) as response:
            payload = json.load(response)
    except urllib.error.HTTPError as exc:
        raise DeployError(
            f"D3 workflow-action Campaign options readback returned HTTP {exc.code}"
        ) from exc
    except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
        raise DeployError("D3 workflow-action Campaign options readback failed") from exc
    if not isinstance(payload, dict) or not isinstance(payload.get("options"), list):
        raise DeployError("D3 workflow-action Campaign options response is invalid")


def derive_ports(
    envv: Mapping[str, str],
    worker_stems: Sequence[str] = LEGACY_WORKER_STEMS,
) -> D3Ports:
    """Reuse ingress ports and allocate private roles deterministically.

    The public test2 nginx configuration already points at BACKEND_PORT and the
    provider adapter port.  Reusing them is what makes this a stopped upgrade,
    not a parallel compatibility deployment.
    """
    backend = require_port(envv.get("BACKEND_PORT", ""), "BACKEND_PORT")
    integration = require_port(
        exp_hubspot_options_adapter_port(dict(envv)),
        "EXP_HUBSPOT_OPTIONS_ADAPTER_PORT",
    )
    stems = tuple(worker_stems)
    if set(stems) not in READABLE_WORKER_STEM_SETS:
        raise DeployError("D3 Runtime Worker role set is unsupported")
    private = [backend + 10, backend + 11, *(backend + 20 + i for i in range(len(stems)))]
    if any(port > 65535 for port in private):
        raise DeployError("test2 BACKEND_PORT leaves no safe D3 private port range")
    worker = {stem: backend + 20 + index for index, stem in enumerate(stems)}
    result = D3Ports(backend, integration, backend + 10, backend + 11, worker)
    values = result.all_ports()
    if len(set(values.values())) != len(values):
        duplicates = sorted(port for port in set(values.values()) if list(values.values()).count(port) > 1)
        raise DeployError(f"D3 test2 port allocation collides: {duplicates}")
    return result


def publication_image(publication: Mapping[str, Any], label: str) -> str:
    repository = publication.get("oci_repository")
    digest = publication.get("digest")
    if not isinstance(repository, str) or not repository.startswith("ghcr.io/pingcap-inc/"):
        raise DeployError(f"{label} has an unexpected OCI repository")
    if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
        raise DeployError(f"{label} is not pinned by an immutable image digest")
    return f"{repository}@{digest}"


def exact_images(bom: Mapping[str, Any]) -> dict[str, str]:
    if is_local_test3_bom(bom):
        errors = validate_deployment_bom_view(dict(bom))
        if errors:
            raise DeployError("invalid Test3 local deployment: " + "; ".join(errors))
        return {role: value["image_id"] for role, value in bom["images"].items()}
    services = bom["services"]
    jobs = bom["jobs"]
    frontend = bom["frontend"]
    return {
        "runtime_api": publication_image(services["runtime_api"], "Runtime API"),
        "runtime_worker": publication_image(services["runtime_worker"], "Runtime Worker"),
        "integration_service": publication_image(services["integration_service"], "Integration Service"),
        "sfcrm_app_service": publication_image(services["sfcrm_app_service"], "sfcrm AppService"),
        **({"mcp_worker": publication_image(services["mcp_worker"], "MCP Worker")} if "mcp_worker" in services else {}),
        "catalog_reconcile": publication_image(jobs["catalog_reconcile"], "Catalog reconcile"),
        "frontend": publication_image(frontend["publication"], "frontend"),
    }


def image_source_commits(bom: Mapping[str, Any]) -> dict[str, str]:
    services = bom["services"]
    result = {
        role: services[role]["source_commit"]
        for role in (
            "runtime_api", "runtime_worker", "integration_service", "sfcrm_app_service",
        )
    }
    if "mcp_worker" in services:
        result["mcp_worker"] = services["mcp_worker"]["source_commit"]
    result.update({
        "catalog_reconcile": bom["jobs"]["catalog_reconcile"]["source_commit"],
        "frontend": bom["frontend"]["publication"]["source_commit"],
    })
    return result


def is_local_test3_bom(bom: Mapping[str, Any]) -> bool:
    return bom.get("schema_version") == LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION


def local_deployment_evidence(bom: Mapping[str, Any]) -> dict[str, str]:
    if not is_local_test3_bom(bom):
        return {}
    return {
        "releaseClass": LOCAL_TEST3_RELEASE_CLASS,
        "controllerCommit": str(bom["controller_commit"]),
    }


def prepare_local_image_candidate(args: argparse.Namespace) -> Path:
    """Freeze local images once for host-local Compose or explicit Test3 use.

    Both entrypoints use the same source, image, AppRelease, metadata and tool
    verification. This envelope carries no registry publication credentials.
    """
    import validate_serviceification_bom as local_bom_contract
    from scripts.verify_serviceification_d1_metadata_authority import VerificationFailure, verify_metadata_authority

    if os.environ.get("GITHUB_ACTIONS", "").lower() == "true":
        raise DeployError("unsigned local candidates are host-only")
    desired = local_bom_contract.load_bom(args.desired_bom.resolve())
    errors = local_bom_contract.validate_bom(desired, require_desired=True)
    if errors:
        raise DeployError("invalid desired BOM: " + "; ".join(errors))
    app = require_exact_checkout(
        args.app_packs_root, desired["app_installation"]["artifact"]["commit"], "AppRelease",
        require_full_history=True,
    )
    l2 = require_exact_checkout(args.l2_root, desired["l2_metadata"]["commit"], "L2 metadata")
    artifact_errors = verify_app_packs_artifacts(desired, app)
    if artifact_errors:
        raise DeployError("exact AppRelease verification failed: " + "; ".join(artifact_errors))
    try:
        verify_metadata_authority(desired, l2, app)
    except VerificationFailure as exc:
        raise DeployError(str(exc)) from exc
    controller = default_capture(("git", "-C", str(REPOSITORY_ROOT), "rev-parse", "HEAD"))
    if default_capture(("git", "-C", str(REPOSITORY_ROOT), "status", "--porcelain")):
        raise DeployError("local deployment controller must be committed before preparation")
    owners = {
        "runtime_api": "pingcap-inc/NexusCRM",
        "runtime_worker": "pingcap-inc/NexusCRM",
        "integration_service": "pingcap-inc/NexusCRM",
        "catalog_reconcile": "pingcap-inc/nexuscrm-controlplane",
        "sfcrm_app_service": "pingcap-inc/nexuscrm-app-packs",
        "frontend": "pingcap-inc/nexuscrm-pingcap-pack",
    }
    images = {}
    for role, owner in owners.items():
        image_id = getattr(args, role + "_image")
        if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
            raise DeployError(f"{role} requires a full local Docker image ID, not a tag")
        inspected = json.loads(default_capture(("docker", "image", "inspect", image_id)))
        if not isinstance(inspected, list) or len(inspected) != 1 or not isinstance(inspected[0], dict):
            raise DeployError(f"{role} local image is unavailable")
        config = inspected[0].get("Config")
        labels = config.get("Labels") if isinstance(config, dict) else None
        if not isinstance(labels, dict):
            raise DeployError(f"{role} local image has no source labels")
        images[role] = {
            "image_id": image_id,
            "source_commit": labels.get("org.opencontainers.image.revision", ""),
            "source_repository": owner,
        }
    candidate = {
        "schema_version": local_bom_contract.LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION,
        "release_class": "test-only-unsigned-non-promotable",
        "controller_commit": controller,
        "desired_bom": desired,
        "desired_bom_sha256": local_bom_contract.canonical_json_sha256(desired),
        "images": images,
    }
    errors = local_bom_contract.validate_local_test3_deployment(candidate)
    if errors:
        raise DeployError("invalid local deployment: " + "; ".join(errors))
    core = require_exact_checkout(args.core_root, images["runtime_api"]["source_commit"], "Core")
    verify_local_test3_images(candidate, roles=tuple(images), capture_output=default_capture)
    approval_source = args.approval_publication.resolve()
    approval = load_json_object(approval_source, "local Approval publication")
    if (
        approval.get("schema") != "nexuscrm.approval-publication-bundle/v1"
        or approval.get("sourceDigest") != desired["l2_metadata"]["tree_sha256"]
        or not isinstance(approval.get("definitions"), list) or not approval["definitions"]
    ):
        raise DeployError("Approval publication does not bind the selected L2 metadata")
    output = args.output_dir.expanduser().absolute()
    if output.exists() or output.is_symlink():
        raise DeployError("local output directory already exists; do not overwrite a frozen candidate")
    output.parent.mkdir(parents=True, exist_ok=True)
    with tempfile.TemporaryDirectory(prefix=".d3-local-", dir=output.parent) as temporary:
        staged = Path(temporary) / "candidate"
        staged.mkdir(mode=0o700)
        candidate_path = staged / "resolved-bom.json"
        candidate_path.write_bytes(local_bom_contract.canonical_json_bytes(candidate))
        tools = args.deployment_tools_root.resolve()
        verify_deployment_tools(root=tools, resolved_bom=candidate_path)
        shutil.copytree(tools, staged / "deployment-tools")
        verify_deployment_tools(root=staged / "deployment-tools", resolved_bom=candidate_path)
        shutil.copyfile(approval_source, staged / "approval-publication.json")
        if load_json_object(staged / "approval-publication.json", "copied Approval publication") != approval:
            raise DeployError("Approval publication changed during preparation")
        # Preserve the input checkouts; this command neither builds nor publishes images.
        require_exact_checkout(core, images["runtime_api"]["source_commit"], "Core")
        verify_local_test3_images(candidate, roles=tuple(images), capture_output=default_capture)
        os.rename(staged, output)
    return output / "resolved-bom.json"



def require_local_image_context(context: D3Context) -> None:
    if os.environ.get("GITHUB_ACTIONS", "").lower() == "true":
        raise DeployError("unsigned local deployment is host-only")
    allowed = {("test-test3", "test3")}
    if (context.envv.get("ENV_NAME"), CONTAINER_LABEL_VALUE) not in allowed:
        raise DeployError("unsigned local deployment is restricted to Test3")


def immutable_image_digest(image: str) -> str:
    return image if SHA256_RE.fullmatch(image) else image.partition("@")[2]


def verify_local_test3_images(
    bom: Mapping[str, Any],
    *,
    capture_output: Callable[[Sequence[str]], str],
    roles: Sequence[str] | None = None,
) -> None:
    """Inspect immutable local bytes; never pull a tag or claim owner signing."""
    projected = operational_bom(dict(bom))
    if not is_local_test3_bom(projected):
        raise DeployError("local image inspection requires the explicit Test3 envelope")
    images = exact_images(projected)
    for role in roles if roles is not None else images:
        expected = projected["images"][role]
        verify_local_role_image(
            role, images[role], expected["source_commit"], expected["source_repository"],
            capture_output=capture_output,
            frontend_source_tuple=(projected["frontend"]["publication"]["source_tuple_sha256"] if role == "frontend" else None),
        )


def verify_local_role_image(
    role: str, image: str, source_commit: str, source_repository: str,
    *, capture_output: Callable[[Sequence[str]], str],
    frontend_source_tuple: str | None = None,
) -> str:
    """Verify local role bytes and return their immutable ID; never pull."""
    if not re.fullmatch(r"sha256:[0-9a-f]{64}", image):
        raise DeployError(f"local {role} requires a full immutable Docker image ID")
    if not re.fullmatch(r"[0-9a-f]{40}", source_commit):
        raise DeployError(f"local {role} requires an exact source revision")
    role_names = {
        "runtime_api": "runtime-api", "runtime_worker": "runtime-worker",
        "integration_service": "integration-service", "sfcrm_app_service": "sfcrm-appservice",
        "catalog_reconcile": "catalog-reconcile", "frontend": "frontend-assets",
    }
    try:
        inspected = json.loads(capture_output(("docker", "image", "inspect", image)))
    except (TypeError, json.JSONDecodeError, subprocess.CalledProcessError) as exc:
        raise DeployError(f"local {role} image inspection is invalid") from exc
    if not isinstance(inspected, list) or len(inspected) != 1 or not isinstance(inspected[0], dict):
        raise DeployError(f"local {role} image is unavailable")
    facts = inspected[0]
    config = facts.get("Config") or {}
    labels = config.get("Labels") or {}
    if (
        facts.get("Id") != image
        or facts.get("Os") != "linux"
        or facts.get("Architecture") != "amd64"
        or labels.get("org.opencontainers.image.revision") != source_commit
    ):
        raise DeployError(f"local {role} image ID/platform/source disagrees with selection")
    source = labels.get("org.opencontainers.image.source")
    if source is not None and source.rstrip("/").removesuffix(".git") != (
        "https://github.com/" + source_repository
    ):
        raise DeployError(f"local {role} image source repository disagrees with selection")
    role_label = {
        "runtime_api": ("dev.nexuscrm.runtime.role", "runtime-api"),
        "runtime_worker": ("dev.nexuscrm.runtime.role", "runtime-worker"),
        "integration_service": ("dev.nexuscrm.integration.role", "integration-service"),
        "sfcrm_app_service": ("dev.nexuscrm.app.role", "sfcrm-app-service"),
        "frontend": ("dev.nexuscrm.frontend.role", "frontend"),
    }.get(role)
    if role_label and role_label[0] in labels and labels[role_label[0]] != role_label[1]:
        raise DeployError(f"local {role} image role label disagrees with selection")
    if role == "frontend":
        if labels.get("io.nexuscrm.frontend.source-tuple-sha256") != frontend_source_tuple:
            raise DeployError("local frontend image source tuple disagrees with selection")
    elif config.get("Entrypoint") != ["/usr/local/bin/" + role_names[role]]:
        raise DeployError(f"local {role} image executable does not match its role")
    return str(facts["Id"])


def pull_and_verify_images(
    context: D3Context,
    roles: Sequence[str],
    *,
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
) -> None:
    """Fetch the exact digest selected and verified by the resolved BOM."""
    if is_local_test3_bom(context.bom):
        require_local_image_context(context)
        verify_local_test3_images(context.bom, roles=roles, capture_output=capture_output)
        return
    for role in roles:
        image = context.images[role]
        execute(("docker", "pull", "--quiet", image))


@dataclass(frozen=True)
class D3Context:
    env_file: Path
    envv: Mapping[str, str]
    env_root: Path
    resolved_bom: Path
    bom: Mapping[str, Any]
    bom_file_digest: str
    run_id: str
    run_root: Path
    state_file: Path
    ports: D3Ports
    images: Mapping[str, str]
    worker_stems: tuple[str, ...] = LEGACY_WORKER_STEMS
    worker_families: tuple[str, ...] = LEGACY_WORKER_FAMILIES
    serving_environment_preparer: Callable[..., dict[str, Any]] | None = None

    def __post_init__(self) -> None:
        object.__setattr__(self, "ports", ports_with_mcp(self.ports, self.images))


@dataclass(frozen=True)
class SealedHistoricalBOM:
    path: Path
    bom: dict[str, Any]
    desired: dict[str, Any]
    images: dict[str, str]


def read_sealed_historical_bom(state: Mapping[str, Any]) -> SealedHistoricalBOM:
    """Read predecessor facts from its run-owned seal, not current admission.

    The original file, desired identity, local authority and exact image set
    must agree with the existing deployment state. Old bytes and grants are
    never rewritten, and this reader does not admit a new deployment.
    """
    run_root = Path(str(state.get("runRoot", ""))).expanduser()
    prepared = state.get("prepared")
    digest = state.get("resolvedBomFileDigest")
    if (
        not run_root.is_absolute() or run_root.is_symlink()
        or not isinstance(prepared, dict)
        or not isinstance(digest, str) or not SHA256_RE.fullmatch(digest)
    ):
        raise DeployError("historical D3 BOM has no sealed run-owned authority")
    raw_path = Path(str(prepared.get("resolvedBomCopy", ""))).expanduser()
    expected = (run_root / "resolved-bom.json").resolve()
    if (
        raw_path.is_symlink() or raw_path.resolve() != expected
        or not expected.is_file()
    ):
        raise DeployError("historical D3 BOM does not match its sealed run-owned copy")
    try:
        serialized = expected.read_bytes()
        if sha256_bytes(serialized) != digest:
            raise DeployError("historical D3 BOM does not match its sealed run-owned copy")
        resolved = parse_bom(serialized, str(expected))
        bom = historical_deployment_bom_view(resolved)
        local = is_local_test3_bom(resolved)
        desired = (
            copy.deepcopy(resolved["desired_bom"]) if local
            else desired_v10_projection_from_resolved(resolved)
        )
        if (
            resolved.get("desired_bom_sha256") != state.get("desiredBomDigest")
            or canonical_json_sha256(desired) != state.get("desiredBomDigest")
            or resolved.get("schema_version")
            != state.get("resolvedBomSchema", RESOLVED_SCHEMA_VERSION_V10)
        ):
            raise DeployError("historical D3 BOM desired identity disagrees with its state")
        if local and (
            (state.get("envName") != "test-test3"
             and not (state.get("envName") == "test-test2" and CONTAINER_LABEL_VALUE == "test3"))
            or any(state.get(key) != value for key, value in local_deployment_evidence(resolved).items())
        ):
            raise DeployError("historical local BOM lost its Test3 controller authority")
        images = (
            {role: image["image_id"] for role, image in resolved["images"].items()}
            if local else exact_images(bom)
        )
        if images != state.get("images"):
            raise DeployError("historical D3 BOM images disagree with their sealed state")
    except (OSError, KeyError, TypeError, ValueError, ValidationFailure) as exc:
        raise DeployError(f"historical D3 BOM is malformed: {exc}") from exc
    return SealedHistoricalBOM(expected, bom, desired, images)


def worker_roles_from_bom(bom: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]:
    roles = bom.get("runtime_worker_roles")
    if not isinstance(roles, list):
        raise DeployError("D3 BOM has no Runtime Worker role authority")
    families = tuple(
        role.get("work_family") if isinstance(role, dict) else None
        for role in roles
    )
    if (
        not families
        or any(not isinstance(family, str) for family in families)
        or len(families) != len(set(families))
        or set(families) not in READABLE_WORKER_FAMILY_SETS
    ):
        raise DeployError("D3 BOM Runtime Worker role authority is unsupported")
    typed_families = tuple(str(family) for family in families)
    return tuple(worker_stem(family) for family in typed_families), typed_families


def state_worker_stems(state: Mapping[str, Any]) -> tuple[str, ...]:
    sealed = state.get("workerStems")
    if sealed is not None:
        if not isinstance(sealed, list) or any(not isinstance(stem, str) for stem in sealed):
            raise DeployError("D3 state has an invalid Runtime Worker role seal")
        stems = tuple(sealed)
    else:
        containers = state.get("containers")
        workers = containers.get("workers") if isinstance(containers, dict) else None
        if isinstance(workers, dict):
            stems = tuple(str(stem) for stem in workers)
        else:
            role_digests = state.get("workerRoleConfigDigests")
            if not isinstance(role_digests, dict):
                raise DeployError("D3 state has no Runtime Worker role authority")
            stems = tuple(str(stem) for stem in role_digests)
    if (
        not stems
        or len(stems) != len(set(stems))
        or set(stems) not in READABLE_WORKER_STEM_SETS
    ):
        raise DeployError("D3 state Runtime Worker role authority is unsupported")
    return stems


def state_worker_families(state: Mapping[str, Any]) -> tuple[str, ...]:
    return tuple(f"runtime.{stem}.v1" for stem in state_worker_stems(state))


def context_worker_stems(context: object) -> tuple[str, ...]:
    stems = getattr(context, "worker_stems", None)
    if isinstance(stems, tuple):
        return stems
    bom = getattr(context, "bom", None)
    if isinstance(bom, dict) and isinstance(bom.get("runtime_worker_roles"), list):
        return worker_roles_from_bom(bom)[0]
    return LEGACY_WORKER_STEMS


@dataclass(frozen=True)
class CatalogAuthority:
    app_installation_id: str
    org_id: str
    active_release_digest: str
    catalog_revision: int
    runtime_audience: str
    security_epoch: int
    active_publication_digest: str
    schema_generation: int
    engine_generation: int


@dataclass(frozen=True)
class CatalogPublisherKey:
    key_id: str
    algorithm: str
    public_key_base64: str
    valid_from: str
    valid_until: str

    def config_value(self) -> dict[str, str]:
        return {
            "keyId": self.key_id,
            "algorithm": self.algorithm,
            "publicKeyBase64": self.public_key_base64,
            "validFrom": self.valid_from,
            "validUntil": self.valid_until,
        }


@dataclass(frozen=True)
class CatalogPublisherTrust:
    publisher_id: str
    ownership_epoch: int
    active: bool
    keys: tuple[CatalogPublisherKey, ...]

    def digest(self) -> str:
        payload = {
            "publisherId": self.publisher_id,
            "active": self.active,
            "keys": [key.config_value() for key in self.keys],
        }
        raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
        return sha256_bytes(raw)


@dataclass(frozen=True)
class WorkerOwnership:
    work_family: str
    deployment_id: str
    worker_release_digest: str
    ownership_epoch: int
    active: bool
    operation_id: str


def decode_hex_text(value: str, label: str) -> str:
    if not value or len(value) % 2 or not re.fullmatch(r"[0-9A-F]+", value):
        raise DeployError(f"authority {label} is not canonical HEX")
    try:
        decoded = bytes.fromhex(value).decode("utf-8")
    except (ValueError, UnicodeDecodeError) as exc:
        raise DeployError(f"authority {label} is not canonical UTF-8") from exc
    if not decoded or decoded.strip() != decoded or any(character in decoded for character in "\r\n\t"):
        raise DeployError(f"authority {label} is not a canonical identifier")
    return decoded


def parse_positive_decimal(value: str, label: str) -> int:
    if not value.isascii() or not value.isdecimal() or value.startswith("0"):
        raise DeployError(f"authority {label} is not a positive canonical decimal")
    parsed = int(value)
    if parsed <= 0 or parsed > 2**64 - 1:
        raise DeployError(f"authority {label} is outside uint64")
    return parsed


def canonical_catalog_time(value: str, label: str) -> str:
    try:
        parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ").replace(
            tzinfo=timezone.utc,
        )
    except ValueError as exc:
        raise DeployError(f"Catalog publisher {label} is not a database UTC timestamp") from exc
    whole = parsed.strftime("%Y-%m-%dT%H:%M:%S")
    fraction = f"{parsed.microsecond:06d}".rstrip("0")
    return whole + (f".{fraction}" if fraction else "") + "Z"


def canonical_catalog_config_time(value: str, label: str) -> str:
    if not re.fullmatch(
        r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z",
        value,
    ):
        raise DeployError(f"Catalog publisher {label} is not canonical UTC RFC3339")
    try:
        parsed = datetime.fromisoformat(value[:-1] + "+00:00")
    except ValueError as exc:
        raise DeployError(
            f"Catalog publisher {label} is not canonical UTC RFC3339",
        ) from exc
    whole = parsed.strftime("%Y-%m-%dT%H:%M:%S")
    fraction = f"{parsed.microsecond:06d}".rstrip("0")
    canonical = whole + (f".{fraction}" if fraction else "") + "Z"
    if canonical != value:
        raise DeployError(f"Catalog publisher {label} is not canonical UTC RFC3339")
    return canonical


def read_catalog_publisher_trust(
    envv: Mapping[str, str],
    *,
    query: Callable[..., str] = db_query_scalar,
) -> CatalogPublisherTrust:
    raw = query(
        dict(envv), CATALOG_PUBLISHER_TRUST_SQL, database=envv["TIDB_DATABASE"],
    )
    rows = raw.splitlines() if raw else []
    if not rows:
        raise DeployError("Catalog publisher io.nexuscrm has no active trust authority")
    publisher_id = ""
    ownership_epoch = 0
    active = False
    keys: list[CatalogPublisherKey] = []
    seen: set[str] = set()
    for row in rows:
        fields = row.split("\t")
        if len(fields) != 8:
            raise DeployError("Catalog publisher trust query returned an unknown shape")
        row_publisher = decode_hex_text(fields[0], "PublisherId")
        row_epoch = parse_positive_decimal(fields[1], "publisher OwnershipEpoch")
        row_active = fields[2] == "ACTIVE"
        if fields[2] not in {"ACTIVE", "REVOKED"}:
            raise DeployError("Catalog publisher has an unknown state")
        if not publisher_id:
            publisher_id, ownership_epoch, active = row_publisher, row_epoch, row_active
        elif (row_publisher, row_epoch, row_active) != (
            publisher_id, ownership_epoch, active,
        ):
            raise DeployError("Catalog publisher trust rows disagree on publisher authority")
        key_id = decode_hex_text(fields[3], "PublisherKey.KeyId")
        if key_id in seen:
            raise DeployError("Catalog publisher trust has a duplicate active key")
        seen.add(key_id)
        algorithm = decode_hex_text(fields[4], "PublisherKey.Algorithm")
        try:
            public_key = bytes.fromhex(fields[5])
        except ValueError as exc:
            raise DeployError("Catalog publisher public key is not HEX") from exc
        if not public_key:
            raise DeployError("Catalog publisher public key is empty")
        keys.append(CatalogPublisherKey(
            key_id=key_id,
            algorithm=algorithm,
            public_key_base64=base64.b64encode(public_key).decode("ascii"),
            valid_from=canonical_catalog_time(fields[6], "ValidFrom"),
            valid_until=canonical_catalog_time(fields[7], "ValidUntil"),
        ))
    if publisher_id != "io.nexuscrm" or not active:
        raise DeployError("Catalog publisher io.nexuscrm is not the active authority")
    return CatalogPublisherTrust(
        publisher_id=publisher_id,
        ownership_epoch=ownership_epoch,
        active=active,
        keys=tuple(sorted(keys, key=lambda key: key.key_id)),
    )


def read_catalog_publisher_transition_receipt(
    envv: Mapping[str, str], operation_id: str,
    *,
    query: Callable[..., str] = db_query_scalar,
) -> dict[str, str] | None:
    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,254}", operation_id):
        raise DeployError("Catalog publisher transition operation ID is not canonical")
    literal = f"CONVERT(X'{operation_id.encode('utf-8').hex()}' USING utf8mb4)"
    raw = query(
        dict(envv),
        CATALOG_PUBLISHER_TRANSITION_RECEIPT_SQL.format(operation_id=literal),
        database=envv["TIDB_DATABASE"],
    )
    rows = raw.splitlines() if raw else []
    if not rows:
        return None
    if len(rows) != 1:
        raise DeployError("Catalog publisher transition has duplicate durable receipts")
    fields = rows[0].split("\t")
    if len(fields) != 6:
        raise DeployError("Catalog publisher transition receipt has an unknown shape")
    return {
        "operationId": decode_hex_text(fields[0], "PublisherTransition.OperationId"),
        "publisherId": decode_hex_text(fields[1], "PublisherTransition.PublisherId"),
        "expectedOwnershipEpoch": str(parse_positive_decimal(
            fields[2], "PublisherTransition.ExpectedOwnershipEpoch",
        )),
        "ownershipEpoch": str(parse_positive_decimal(
            fields[3], "PublisherTransition.ResultOwnershipEpoch",
        )),
        "previousTrustDigest": fields[4],
        "activeTrustDigest": fields[5],
    }


def required_tables_present(
    envv: Mapping[str, str],
    required: set[str],
    sql: str,
    *,
    query: Callable[..., str] = db_query_scalar,
) -> bool:
    raw = query(dict(envv), sql, database=envv["TIDB_DATABASE"])
    actual = set(raw.splitlines()) if raw else set()
    if actual == required:
        return True
    if not actual:
        return False
    raise DeployError(
        "D3 found a partial authority schema: present="
        f"{sorted(actual)} required={sorted(required)}"
    )


def catalog_authority_tables_present(envv: Mapping[str, str]) -> bool:
    return required_tables_present(
        envv, CATALOG_AUTHORITY_TABLES, CATALOG_AUTHORITY_TABLES_SQL,
    )


def runtime_work_ownership_table_present(envv: Mapping[str, str]) -> bool:
    return required_tables_present(
        envv, RUNTIME_WORK_OWNERSHIP_TABLES, RUNTIME_WORK_OWNERSHIP_TABLES_SQL,
    )


def parse_catalog_authority_row(row: str) -> CatalogAuthority:
    fields = row.split("\t")
    if len(fields) != 10:
        raise DeployError("Catalog authority query returned an unknown shape")
    (
        installation_hex,
        org_hex,
        active_release,
        installation_revision,
        audience_hex,
        org_revision,
        security_epoch,
        publication_digest,
        schema_generation,
        engine_generation,
    ) = fields
    if not SHA256_RE.fullmatch(active_release) or not SHA256_RE.fullmatch(publication_digest):
        raise DeployError("Catalog authority has a malformed release or publication digest")
    # An installation records the revision of its own last release operation.
    # The org writer lineage can legitimately be newer after another app or
    # source publishes. Validate both stored revisions, then use the org's
    # active publication as the serving and next-operation authority.
    parse_positive_decimal(
        installation_revision, "AppInstallation CatalogRevision",
    )
    org_revision_value = parse_positive_decimal(org_revision, "org CatalogRevision")
    return CatalogAuthority(
        app_installation_id=decode_hex_text(installation_hex, "AppInstallationId"),
        org_id=decode_hex_text(org_hex, "OrgId"),
        active_release_digest=active_release,
        catalog_revision=org_revision_value,
        runtime_audience=decode_hex_text(audience_hex, "RuntimeAudience"),
        security_epoch=parse_positive_decimal(security_epoch, "SecurityEpoch"),
        active_publication_digest=publication_digest,
        schema_generation=parse_positive_decimal(schema_generation, "SchemaGeneration"),
        engine_generation=parse_positive_decimal(engine_generation, "EngineGeneration"),
    )


def read_catalog_authority(
    envv: Mapping[str, str],
    *,
    query: Callable[..., str] = db_query_scalar,
) -> CatalogAuthority:
    raw = query(dict(envv), CATALOG_AUTHORITY_SQL, database=envv["TIDB_DATABASE"])
    rows = raw.splitlines() if raw else []
    if len(rows) != 1:
        raise DeployError(
            "D3 requires exactly one idle ACTIVE io.nexuscrm.sfcrm installation bound "
            "to the current immutable Catalog publication"
        )
    return parse_catalog_authority_row(rows[0])


def read_optional_catalog_authority(
    envv: Mapping[str, str],
    *,
    tables_present: Callable[[Mapping[str, str]], bool] = catalog_authority_tables_present,
    query: Callable[..., str] = db_query_scalar,
) -> CatalogAuthority | None:
    if not tables_present(envv):
        return None
    raw = query(dict(envv), CATALOG_AUTHORITY_SQL, database=envv["TIDB_DATABASE"])
    rows = raw.splitlines() if raw else []
    if not rows:
        return None
    if len(rows) != 1:
        raise DeployError("D3 Catalog authority is ambiguous")
    return parse_catalog_authority_row(rows[0])


def read_optional_unpublished_upgrade_authority(
    envv: Mapping[str, str],
    operation_id: str,
    *,
    tables_present: Callable[[Mapping[str, str]], bool] = catalog_authority_tables_present,
    query: Callable[..., str] = db_query_scalar,
) -> CatalogAuthority | None:
    """Read the still-active authority hidden behind one unpublished upgrade."""
    if not tables_present(envv):
        return None
    if not re.fullmatch(
        r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
        operation_id,
    ):
        raise DeployError("Catalog unpublished upgrade operation identity is malformed")
    operation_literal = (
        f"CONVERT(X'{operation_id.encode('utf-8').hex()}' USING utf8mb4)"
    )
    sql = f"""
SELECT CONCAT_WS('\t',
  HEX(i.AppInstallationId), HEX(i.OrgId), i.ActiveReleaseDigest,
  CAST(i.CatalogRevision AS CHAR), HEX(s.RuntimeAudience),
  CAST(s.CatalogRevision AS CHAR), CAST(s.LastSecurityEpoch AS CHAR),
  a.ActivePublicationDigest, CAST(p.SchemaGeneration AS CHAR),
  CAST(p.EngineGeneration AS CHAR))
FROM _System_CatalogAppInstallation i
JOIN _System_CatalogAppOperation o ON o.OperationId = i.ActiveOperationId
JOIN _System_CatalogOrgState s ON s.OrgId = i.OrgId
JOIN _System_CatalogPublicationActive a ON a.OrgId = i.OrgId
JOIN _System_CatalogPublication p
  ON p.OrgId = i.OrgId AND p.PublicationDigest = a.ActivePublicationDigest
WHERE i.AppId = 'io.nexuscrm.sfcrm'
  AND i.State = 'UPGRADING'
  AND i.ActiveReleaseDigest IS NOT NULL
  AND i.ActiveOperationId = {operation_literal}
  AND s.ActiveOperationId = i.ActiveOperationId
  AND o.OrgId = i.OrgId AND o.AppId = i.AppId
  AND o.Kind = 'UPGRADE' AND o.State = 'UPGRADING'
  AND o.Step = 'CANDIDATE_COMMITTED'
  AND NOT EXISTS (
    SELECT 1 FROM _System_CatalogAppOperationJournal j
    WHERE j.OperationId = o.OperationId
      AND j.Step IN ('METADATA_SELECTION_COMMITTED',
        'SERVING_ARTIFACTS_COMMITTED', 'PROJECTION_ACKNOWLEDGED', 'ACTIVATED'))
  AND i.CatalogRevision = s.CatalogRevision
  AND a.ActivePublicationPresent = 1
  AND a.ActivePublicationDigest IS NOT NULL
  AND p.CatalogRevision = i.CatalogRevision
  AND p.SecurityEpoch = s.LastSecurityEpoch;
""".strip()
    raw = query(dict(envv), sql, database=envv["TIDB_DATABASE"])
    rows = raw.splitlines() if raw else []
    if not rows:
        return None
    if len(rows) != 1:
        raise DeployError("Catalog unpublished upgrade authority is ambiguous")
    return parse_catalog_authority_row(rows[0])


def read_optional_catalog_abandonment_receipt(
    envv: Mapping[str, str],
    operation_id: str,
    evidence_digest: str,
    *,
    tables_present: Callable[[Mapping[str, str]], bool] = catalog_authority_tables_present,
    query: Callable[..., str] = db_query_scalar,
) -> bool:
    """Prove one exact abandonment committed without publication journal steps."""
    if not tables_present(envv):
        return False
    if not re.fullmatch(
        r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
        operation_id,
    ) or not SHA256_RE.fullmatch(evidence_digest):
        raise DeployError("Catalog abandonment receipt identity is malformed")
    operation_literal = (
        f"CONVERT(X'{operation_id.encode('utf-8').hex()}' USING utf8mb4)"
    )
    sql = f"""
SELECT CONCAT_WS('\t', HEX(o.OperationId), o.State, o.Step, o.LastError,
  CAST(COALESCE(SUM(j.Step = 'CANDIDATE_COMMITTED'), 0) AS CHAR),
  CAST(COALESCE(SUM(j.Step = 'ABANDONED'), 0) AS CHAR),
  CAST(COALESCE(SUM(j.Step IN ('METADATA_SELECTION_COMMITTED',
    'SERVING_ARTIFACTS_COMMITTED', 'PROJECTION_ACKNOWLEDGED', 'ACTIVATED')), 0)
    AS CHAR))
FROM _System_CatalogAppOperation o
LEFT JOIN _System_CatalogAppOperationJournal j ON j.OperationId = o.OperationId
WHERE o.OperationId = {operation_literal}
GROUP BY o.OperationId, o.State, o.Step, o.LastError;
""".strip()
    raw = query(dict(envv), sql, database=envv["TIDB_DATABASE"])
    rows = raw.splitlines() if raw else []
    if not rows:
        return False
    if len(rows) != 1:
        raise DeployError("Catalog abandonment has duplicate durable receipts")
    fields = rows[0].split("\t")
    if len(fields) != 7:
        raise DeployError("Catalog abandonment receipt has an unknown shape")
    if (
        decode_hex_text(fields[0], "Catalog abandonment OperationId") != operation_id
        or fields[1:3] != ["FAILED", "ABANDONED"]
        or fields[3] != evidence_digest
        or fields[4:] != ["1", "1", "0"]
    ):
        raise DeployError("Catalog abandonment durable receipt is inconsistent")
    return True


def read_worker_ownership(
    envv: Mapping[str, str],
    *,
    query: Callable[..., str] = db_query_scalar,
) -> dict[str, WorkerOwnership | None]:
    raw = query(dict(envv), WORKER_OWNERSHIP_SQL, database=envv["TIDB_DATABASE"])
    result: dict[str, WorkerOwnership | None] = {
        family: None for family in SUPPORTED_WORKER_FAMILIES
    }
    for row in raw.splitlines() if raw else []:
        fields = row.split("\t")
        if len(fields) != 6:
            raise DeployError("Runtime Worker ownership query returned an unknown shape")
        family = decode_hex_text(fields[0], "Runtime Worker WorkFamily")
        if family not in result or result[family] is not None:
            raise DeployError("Runtime Worker ownership contains an unknown or duplicate family")
        active_text = fields[4]
        if active_text not in {"0", "1"}:
            raise DeployError("Runtime Worker ownership IsActive is not canonical")
        ownership = WorkerOwnership(
            work_family=family,
            deployment_id=decode_hex_text(fields[1], "Runtime Worker DeploymentId"),
            worker_release_digest=fields[2],
            ownership_epoch=parse_positive_decimal(fields[3], "Runtime Worker OwnershipEpoch"),
            active=active_text == "1",
            operation_id=decode_hex_text(fields[5], "Runtime Worker OperationId"),
        )
        if not SHA256_RE.fullmatch(ownership.worker_release_digest):
            raise DeployError("Runtime Worker ownership release digest is malformed")
        result[family] = ownership
    return result


def read_optional_worker_ownership(
    envv: Mapping[str, str],
    *,
    table_present: Callable[[Mapping[str, str]], bool] = runtime_work_ownership_table_present,
) -> tuple[bool, dict[str, WorkerOwnership | None]]:
    if not table_present(envv):
        return False, {family: None for family in SUPPORTED_WORKER_FAMILIES}
    return True, read_worker_ownership(envv)


def worker_ownership_payload(
    ownership: Mapping[str, WorkerOwnership | None],
) -> dict[str, dict[str, Any] | None]:
    if set(ownership) != set(SUPPORTED_WORKER_FAMILIES):
        raise DeployError("Runtime Worker ownership snapshot is incomplete")
    return {
        family: None if ownership[family] is None else {
            "workFamily": ownership[family].work_family,
            "deploymentId": ownership[family].deployment_id,
            "workerReleaseDigest": ownership[family].worker_release_digest,
            "ownershipEpoch": ownership[family].ownership_epoch,
            "active": ownership[family].active,
            "operationId": ownership[family].operation_id,
        }
        for family in sorted(ownership)
    }


def require_worker_ownership_before(
    state: Mapping[str, Any],
) -> dict[str, WorkerOwnership | None]:
    value = state.get("workerOwnershipBefore")
    if (
        not isinstance(value, dict)
        or set(value) not in READABLE_WORKER_FAMILY_SETS
    ):
        raise DeployError("quiesced D3 state has no complete Runtime Worker ownership boundary")
    value = {**{family: None for family in SUPPORTED_WORKER_FAMILIES}, **value}
    result: dict[str, WorkerOwnership | None] = {}
    for family in SUPPORTED_WORKER_FAMILIES:
        item = value[family]
        if item is None:
            result[family] = None
            continue
        if not isinstance(item, dict) or set(item) != {
            "workFamily", "deploymentId", "workerReleaseDigest",
            "ownershipEpoch", "active", "operationId",
        }:
            raise DeployError("Runtime Worker ownership boundary has an unknown shape")
        ownership = WorkerOwnership(
            work_family=str(item["workFamily"]),
            deployment_id=str(item["deploymentId"]),
            worker_release_digest=str(item["workerReleaseDigest"]),
            ownership_epoch=item["ownershipEpoch"],
            active=item["active"],
            operation_id=str(item["operationId"]),
        )
        if (
            ownership.work_family != family
            or not ownership.deployment_id
            or not SHA256_RE.fullmatch(ownership.worker_release_digest)
            or not isinstance(ownership.ownership_epoch, int)
            or isinstance(ownership.ownership_epoch, bool)
            or ownership.ownership_epoch <= 0
            or not isinstance(ownership.active, bool)
            or not ownership.operation_id
        ):
            raise DeployError("Runtime Worker ownership boundary is incomplete")
        result[family] = ownership
    return result


def load_context(env_file: Path, resolved_bom: Path, run_id: str) -> D3Context:
    if not RUN_ID_RE.fullmatch(run_id):
        raise DeployError("D3 run id must be one bounded canonical identifier")
    env_file = env_file.expanduser().resolve()
    envv = load_env_file(env_file)
    if envv.get("ENV_NAME") != "test-test2":
        raise DeployError("D3 stopped-upgrade runner is restricted to ENV_NAME=test-test2")
    if not bool_env(envv, "INSTALL_SYSTEMD", True):
        raise DeployError("D3 stopped upgrade requires the test2 systemd deployment")
    env_root_text = envv.get("ENV_ROOT", "").strip()
    if not env_root_text:
        raise DeployError("test2 runtime env has no ENV_ROOT")
    env_root = Path(env_root_text).expanduser().resolve()
    expected_env_file = runtime_paths(env_root)["runtime_env_file"].resolve()
    if env_file != expected_env_file:
        raise DeployError(
            f"D3 runtime env must be the active test2 authority file: {expected_env_file}"
        )
    resolved_bom = resolved_bom.expanduser().resolve()
    bom = load_bom(resolved_bom)
    if is_local_test3_bom(bom):
        raise DeployError("local deployment requires the explicit Test3 environment adapter")
    serialized = resolved_bom.read_bytes()
    state_root = env_root / ".runtime" / "serviceification-d3"
    run_root = state_root / "runs" / run_id
    deployment_bom = operational_bom(bom)
    worker_stems, worker_families = worker_roles_from_bom(deployment_bom)
    return D3Context(
        env_file=env_file,
        envv=envv,
        env_root=env_root,
        resolved_bom=resolved_bom,
        bom=deployment_bom,
        bom_file_digest=sha256_bytes(serialized),
        run_id=run_id,
        run_root=run_root,
        state_file=state_root / "active.json",
        ports=derive_ports(envv, worker_stems),
        images=exact_images(deployment_bom),
        worker_stems=worker_stems,
        worker_families=worker_families,
    )


def load_active_transition_context(
    env_file: Path,
    run_id: str,
    *,
    allow_legacy_reset_receipt: bool = False,
) -> D3Context:
    """Load the sealed BOM owned by an already-active Test2 transition."""
    if not RUN_ID_RE.fullmatch(run_id):
        raise DeployError("D3 run id must be one bounded canonical identifier")
    env_file = env_file.expanduser().resolve()
    envv = load_env_file(env_file)
    env_root_text = envv.get("ENV_ROOT", "").strip()
    if envv.get("ENV_NAME") != "test-test2" or not env_root_text:
        raise DeployError("active D3 recovery is restricted to the test2 authority")
    if not bool_env(envv, "INSTALL_SYSTEMD", True):
        raise DeployError("active D3 recovery requires the test2 systemd deployment")
    env_root = Path(env_root_text).expanduser().resolve()
    expected_env_file = runtime_paths(env_root)["runtime_env_file"].resolve()
    if env_file != expected_env_file:
        raise DeployError(
            f"active D3 recovery requires the canonical test2 runtime env: {expected_env_file}"
        )
    state_root = env_root / ".runtime" / "serviceification-d3"
    state = read_state(state_root / "active.json")
    local_state = state.get("resolvedBomSchema") == LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION
    if local_state and (
        CONTAINER_LABEL_VALUE != "test3"
        or state.get("releaseClass") != LOCAL_TEST3_RELEASE_CLASS
        or not re.fullmatch(r"[0-9a-f]{40}", str(state.get("controllerCommit", "")))
    ):
        raise DeployError("local deployment recovery requires its Test3-only sealed authority")
    run_root = state_root / "runs" / run_id
    if (
        state.get("envName") != "test-test2"
        or state.get("envRoot") != str(env_root)
        or state.get("runId") != run_id
        or state.get("runRoot") != str(run_root)
    ):
        raise DeployError("active D3 recovery target does not match the requested run")
    prepared = state.get("prepared")
    if not isinstance(prepared, dict):
        # Capture is itself a sealed pre-mutation recovery boundary.  The BOM
        # copy and prepared binaries are intentionally created by the next
        # phase, so requiring them here makes the documented captured-state
        # recovery impossible.  Recovery only archives this state; it does not
        # execute or adopt caller-provided artifacts.
        if state.get("phase") not in {
            "captured", "catalog-successor-captured", "runtime-successor-captured",
        }:
            raise DeployError("active D3 recovery target has no sealed prepared artifacts")
        desired_bom_digest = state.get("desiredBomDigest")
        bom_file_digest = state.get("resolvedBomFileDigest")
        images = state.get("images")
        worker_stems = state_worker_stems(state)
        worker_families = tuple(f"runtime.{stem}.v1" for stem in worker_stems)
        ports = ports_with_mcp(derive_ports(envv, worker_stems), images if isinstance(images, dict) else {})
        expected_image_roles = {
            "runtime_api", "runtime_worker", "integration_service",
            "sfcrm_app_service", "catalog_reconcile", "frontend",
        }
        if (
            not isinstance(desired_bom_digest, str)
            or not SHA256_RE.fullmatch(desired_bom_digest)
            or not isinstance(bom_file_digest, str)
            or not SHA256_RE.fullmatch(bom_file_digest)
            or not isinstance(images, dict)
            or set(images) != (expected_image_roles | ({"mcp_worker"} if "mcp_worker" in images else set()))
            or any(
                not isinstance(image, str)
                or not re.fullmatch(
                    r"sha256:[0-9a-f]{64}" if local_state else
                    r"ghcr[.]io/pingcap-inc/[a-z0-9._/-]+@sha256:[0-9a-f]{64}",
                    image,
                )
                for image in images.values()
            )
            or state.get("ports") != ports.all_ports()
        ):
            raise DeployError("captured D3 recovery identity is incomplete")
        resolved_bom_schema = state.get("resolvedBomSchema", RESOLVED_SCHEMA_VERSION_V10)
        context = D3Context(
            env_file=env_file,
            envv=envv,
            env_root=env_root,
            resolved_bom=Path(str(state.get("resolvedBomPath", ""))).expanduser(),
            bom={
                "schema_version": resolved_bom_schema,
                "desired_bom_sha256": desired_bom_digest,
                **({
                    "release_class": LOCAL_TEST3_RELEASE_CLASS,
                    "controller_commit": state["controllerCommit"],
                } if local_state else {}),
            },
            bom_file_digest=bom_file_digest,
            run_id=run_id,
            run_root=run_root,
            state_file=state_root / "active.json",
            ports=ports,
            images=dict(images),
            worker_stems=worker_stems,
            worker_families=worker_families,
        )
        require_state(
            context, expected_phase={
                "captured", "catalog-successor-captured", "runtime-successor-captured",
            },
        )
        return context
    raw_resolved_bom = Path(str(prepared.get("resolvedBomCopy", ""))).expanduser()
    if raw_resolved_bom.is_symlink():
        raise DeployError("active D3 recovery target lost its sealed resolved BOM")
    resolved_bom = raw_resolved_bom.resolve()
    expected_bom = (run_root / "resolved-bom.json").resolve()
    if (
        resolved_bom != expected_bom
        or not resolved_bom.is_file()
        or sha256_bytes(resolved_bom.read_bytes()) != state.get("resolvedBomFileDigest")
    ):
        raise DeployError("active D3 recovery target lost its sealed resolved BOM")
    history = read_sealed_historical_bom(state)
    bom = history.bom
    if is_local_test3_bom(bom) != local_state:
        raise DeployError("active D3 recovery changed its local/release authority kind")
    desired_bom_digest = bom.get("desired_bom_sha256")
    if (
        not isinstance(desired_bom_digest, str)
        or not SHA256_RE.fullmatch(desired_bom_digest)
    ):
        raise DeployError("active D3 recovery target has an invalid BOM identity")
    try:
        expected_desired_bom_digest = canonical_json_sha256(history.desired)
    except (KeyError, TypeError) as exc:
        raise DeployError("active D3 recovery target has an incomplete BOM") from exc
    if desired_bom_digest != expected_desired_bom_digest:
        raise DeployError("active D3 recovery target has an inconsistent BOM identity")
    try:
        deployment_bom = history.bom
        images = history.images
        worker_stems, worker_families = worker_roles_from_bom(deployment_bom)
        ports = ports_with_mcp(derive_ports(envv, worker_stems), images)
    except (KeyError, TypeError) as exc:
        raise DeployError("active D3 recovery target has an incomplete BOM") from exc
    if state.get("images") != images or state.get("ports") != ports.all_ports():
        raise DeployError("active D3 recovery target does not match its sealed serving identity")
    context = D3Context(
        env_file=env_file,
        envv=envv,
        env_root=env_root,
        resolved_bom=resolved_bom,
        bom=deployment_bom,
        bom_file_digest=sha256_bytes(resolved_bom.read_bytes()),
        run_id=run_id,
        run_root=run_root,
        state_file=state_root / "active.json",
        ports=ports,
        images=images,
        worker_stems=worker_stems,
        worker_families=worker_families,
    )
    require_state(context, allow_legacy_reset_receipt=allow_legacy_reset_receipt)
    return context


def container_name(run_id: str, role: str) -> str:
    normalized = re.sub(r"[^a-z0-9_.-]+", "-", f"nexuscrm-d3-test2-{run_id}-{role}".lower())
    if len(normalized) > 128:
        suffix = hashlib.sha256(normalized.encode()).hexdigest()[:16]
        normalized = normalized[:111].rstrip("-.") + "-" + suffix
    return normalized


def snapshot_legacy_units(envv: Mapping[str, str]) -> dict[str, dict[str, str]]:
    return {
        unit: staging.systemd_unit_state(unit)
        for unit in staging.staging_target_systemd_services(dict(envv))
    }


def restore_legacy_units(
    states: Mapping[str, Mapping[str, str]],
    *,
    execute: Callable[[Sequence[str]], None],
) -> None:
    """Restore the exact pre-D3 enable/start shape after Catalog rollback."""
    for unit, state in states.items():
        if state.get("load_state") == "not-found":
            continue
        unit_file_state = state.get("unit_file_state")
        if unit_file_state == "enabled":
            execute(("sudo", "-n", "systemctl", "enable", unit))
        elif unit_file_state == "enabled-runtime":
            execute(("sudo", "-n", "systemctl", "enable", "--runtime", unit))
        elif unit_file_state == "static":
            pass
        elif unit_file_state in {"disabled", "masked", "masked-runtime"}:
            if unit_file_state == "disabled":
                execute(("sudo", "-n", "systemctl", "disable", unit))
        else:
            raise DeployError(f"cannot restore unknown systemd state for {unit}: {unit_file_state}")
    # Start dependencies before adapters. staging_target_systemd_services is
    # adapter, backend; reversing it restores the backend owner first.
    for unit, state in reversed(tuple(states.items())):
        if state.get("load_state") == "not-found":
            continue
        if state.get("active_state") == "active":
            execute(("sudo", "-n", "systemctl", "start", unit))
        elif state.get("active_state") in {"inactive", "failed"}:
            execute(("sudo", "-n", "systemctl", "stop", unit))
        else:
            raise DeployError(
                f"cannot restore non-terminal systemd state for {unit}: {state.get('active_state')}"
            )


def state_payload(
    context: D3Context,
    *,
    previous_frontend_target: Path | None = None,
    previous_systemd_units: Mapping[str, Mapping[str, str]] | None = None,
) -> dict[str, Any]:
    frontend_target = previous_frontend_target
    if frontend_target is None:
        frontend_target = resolve_frontend_dist_target(
            runtime_paths(context.env_root)["frontend_dist"]
        )
    systemd_units = previous_systemd_units
    if systemd_units is None:
        systemd_units = snapshot_legacy_units(context.envv)
    return {
        "schema": STATE_SCHEMA,
        "phase": "captured",
        "envName": context.envv["ENV_NAME"],
        "envRoot": str(context.env_root),
        "runId": context.run_id,
        "runRoot": str(context.run_root),
        "resolvedBomPath": str(context.resolved_bom),
        "resolvedBomSchema": context.bom["schema_version"],
        "resolvedBomFileDigest": context.bom_file_digest,
        "desiredBomDigest": context.bom["desired_bom_sha256"],
        **local_deployment_evidence(context.bom),
        "images": dict(context.images),
        "ports": context.ports.all_ports(),
        "workerStems": list(context.worker_stems),
        "previousFrontendTarget": str(frontend_target),
        "previousSystemdUnits": dict(systemd_units),
        "containers": {
            "runtime_api": container_name(context.run_id, "runtime-api"),
            "integration_service": container_name(context.run_id, "integration"),
            "sfcrm_app_service": container_name(context.run_id, "sfcrm"),
            **({"mcp_worker": container_name(context.run_id, "mcp")} if "mcp_worker" in context.images else {}),
            "workers": {
                stem: container_name(context.run_id, f"worker-{stem}")
                for stem in context.worker_stems
            },
        },
    }


def write_new_state(context: D3Context, payload: Mapping[str, Any]) -> None:
    if context.state_file.exists() or context.state_file.is_symlink():
        raise DeployError(
            f"D3 test2 already has an active transition state: {context.state_file}"
        )
    context.run_root.mkdir(parents=True, mode=0o700, exist_ok=False)
    context.state_file.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
    pending = context.state_file.with_suffix(".json.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError(f"stale D3 state publication exists: {pending}")
    pending.write_bytes(canonical_bytes(dict(payload)))
    os.chmod(pending, 0o600)
    os.replace(pending, context.state_file)


def replace_state(
    context: D3Context,
    *,
    expected_phase: str,
    updates: Mapping[str, Any],
    allow_legacy_reset_receipt: bool = False,
) -> dict[str, Any]:
    current = require_state(
        context,
        expected_phase=expected_phase,
        allow_legacy_reset_receipt=allow_legacy_reset_receipt,
    )
    next_value = {**current, **dict(updates)}
    pending = context.state_file.with_suffix(".json.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError(f"stale D3 state publication exists: {pending}")
    pending.write_bytes(canonical_bytes(next_value))
    os.chmod(pending, 0o600)
    os.replace(pending, context.state_file)
    return next_value


def require_state(
    context: D3Context,
    *,
    expected_phase: str | set[str] | None = None,
    allow_legacy_reset_receipt: bool = False,
) -> dict[str, Any]:
    current = read_state(context.state_file)
    if (
        current.get("envName") != context.envv["ENV_NAME"]
        or current.get("envRoot") != str(context.env_root)
        or current.get("runId") != context.run_id
        or current.get("resolvedBomFileDigest") != context.bom_file_digest
        or current.get("desiredBomDigest") != context.bom["desired_bom_sha256"]
        or current.get("resolvedBomSchema", RESOLVED_SCHEMA_VERSION_V10)
        != context.bom["schema_version"]
        or any(current.get(key) != value for key, value in local_deployment_evidence(context.bom).items())
    ):
        raise DeployError("active D3 state does not belong to this exact run and resolved BOM")
    allowed = (
        {expected_phase} if isinstance(expected_phase, str)
        else expected_phase
    )
    if allowed is not None and current.get("phase") not in allowed:
        raise DeployError(
            f"D3 state phase is {current.get('phase')}, expected one of {sorted(allowed)}"
        )
    if not allow_legacy_reset_receipt:
        require_reset_snapshot_connector_queue_purge_receipt(current)
    return current


def read_state(path: Path) -> dict[str, Any]:
    try:
        value = json.loads(path.read_text())
    except (OSError, json.JSONDecodeError) as exc:
        raise DeployError(f"cannot read D3 state {path}: {exc}") from exc
    if not isinstance(value, dict) or value.get("schema") != STATE_SCHEMA:
        raise DeployError(f"D3 state has an unknown schema: {path}")
    return value


def default_execute(args: Sequence[str]) -> None:
    subprocess.run(list(args), check=True)


def default_capture(args: Sequence[str]) -> str:
    return subprocess.run(
        list(args), check=True, capture_output=True, text=True,
    ).stdout.strip()


def default_capture_combined(args: Sequence[str]) -> str:
    result = subprocess.run(
        list(args), check=False, capture_output=True, text=True,
    )
    if result.returncode != 0:
        raise subprocess.CalledProcessError(
            result.returncode, list(args), output=result.stdout, stderr=result.stderr,
        )
    return (result.stdout + result.stderr).strip()


def frontend_tree_digest(root: Path) -> str:
    """Digest one extracted frontend without trusting its containing directory."""
    if root.is_symlink() or not root.is_dir():
        raise DeployError(f"D3 frontend artifact is unavailable: {root}")
    digest = hashlib.sha256()
    for path in sorted(root.rglob("*"), key=lambda value: value.relative_to(root).as_posix()):
        if path.is_symlink():
            raise DeployError(f"D3 frontend artifact contains a symlink: {path}")
        if path.is_dir():
            continue
        if not path.is_file():
            raise DeployError(f"D3 frontend artifact contains a non-file: {path}")
        relative = path.relative_to(root).as_posix().encode()
        digest.update(len(relative).to_bytes(8, "big"))
        digest.update(relative)
        content = path.read_bytes()
        digest.update(len(content).to_bytes(8, "big"))
        digest.update(content)
    return "sha256:" + digest.hexdigest()


def publish_readable_frontend(context: D3Context, source: Path) -> tuple[Path, str]:
    """Publish an exact frontend outside the private 0700 transition root."""
    return publish_readable_frontend_release(
        context,
        source,
        release_digest=context.bom["frontend"]["publication"]["digest"],
    )


def publish_readable_frontend_release(
    context: D3Context,
    source: Path,
    *,
    release_digest: str,
) -> tuple[Path, str]:
    """Publish one exact frontend content owner without changing Catalog state."""
    verify_frontend_dist(source)
    content_digest = frontend_tree_digest(source)
    release_root = context.env_root / "frontend-releases"
    release_root.mkdir(mode=0o755, parents=True, exist_ok=True)
    os.chmod(release_root, 0o755)
    if not isinstance(release_digest, str) or not SHA256_RE.fullmatch(release_digest):
        raise DeployError("frontend release requires an exact digest")
    image_digest = release_digest.removeprefix("sha256:")
    target = release_root / f"d3-{image_digest}"
    if target.is_symlink():
        raise DeployError(f"D3 frontend publication target is a symlink: {target}")
    if not target.exists():
        pending = release_root / f".{target.name}.{context.run_id}.pending"
        if pending.exists() or pending.is_symlink():
            raise DeployError(f"stale D3 frontend publication exists: {pending}")
        shutil.copytree(source, pending)
        for path in sorted(pending.rglob("*"), reverse=True):
            os.chmod(path, 0o555 if path.is_dir() else 0o444)
        os.chmod(pending, 0o555)
        os.replace(pending, target)
    verify_frontend_dist(target)
    if frontend_tree_digest(target) != content_digest:
        raise DeployError("published D3 frontend content disagrees with the exact image artifact")
    if context.envv.get("ENV_NAME", "").startswith("multi-prod-d3-"):
        from deployments.online.runtime.prod_frontend_acl import ensure_frontend_acl
        try:
            backup = ensure_frontend_acl(context.env_root, target)
        except (OSError, ValueError, KeyError, subprocess.SubprocessError) as exc:
            raise DeployError("Prod frontend ACL/access check failed before cutover") from exc
        if backup:
            print(f"[deploy] Prod frontend ACL backup: {backup}")
    live_frontend = runtime_paths(context.env_root)["frontend_dist"]
    if resolve_frontend_dist_target(live_frontend) != target.resolve():
        publish_symlink(live_frontend, target)
    if resolve_frontend_dist_target(live_frontend) != target.resolve():
        raise DeployError("D3 frontend publication did not bind the readable exact artifact")
    return target.resolve(), content_digest


def fetch_frontend_version(url: str, headers: Mapping[str, str]) -> str:
    """Read the bounded public frontend identity instead of trusting local state."""
    request = urllib.request.Request(url, headers=dict(headers), method="GET")
    try:
        with urllib.request.urlopen(request, timeout=10) as response:
            payload = response.read(64 * 1024 + 1)
    except (OSError, TimeoutError) as exc:
        raise DeployError(f"frontend serving identity is unavailable at {url}: {exc}") from exc
    if len(payload) > 64 * 1024:
        raise DeployError(f"frontend serving identity is oversized at {url}")
    try:
        decoded = json.loads(payload)
    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
        raise DeployError(f"frontend serving identity is malformed at {url}") from exc
    version = decoded.get("version") if isinstance(decoded, dict) else None
    if not isinstance(version, str) or not SHA256_RE.fullmatch(version):
        raise DeployError(f"frontend serving identity has no exact version at {url}")
    return version


def verify_served_frontend_version(
    context: D3Context,
    expected_version: str,
    *,
    fetch: Callable[[str, Mapping[str, str]], str] = fetch_frontend_version,
    attempts: int = 5,
) -> dict[str, str]:
    """Require both the local nginx owner and the public origin to serve one release."""
    if not SHA256_RE.fullmatch(expected_version):
        raise DeployError("frontend serving verification requires an exact version")
    if attempts < 1:
        raise DeployError("frontend serving verification requires at least one attempt")
    cache_key = expected_version.removeprefix("sha256:")
    host = context.envv["SERVER_NAME"].split()[0]
    local_url = (
        f"http://127.0.0.1:{context.envv.get('NGINX_LISTEN_PORT', '80')}"
        f"/version.json?nexuscrmFrontendActivation={cache_key}"
    )
    public_url = (
        context.envv["PUBLIC_BASE_URL"].rstrip("/")
        + f"/version.json?nexuscrmFrontendActivation={cache_key}"
    )
    observed = {"local": "unavailable", "public": "unavailable"}
    last_error: DeployError | None = None
    for attempt in range(attempts):
        try:
            observed = {
                "local": fetch(local_url, {"Host": host, "Cache-Control": "no-cache"}),
                "public": fetch(public_url, {"Cache-Control": "no-cache"}),
            }
            if all(value == expected_version for value in observed.values()):
                return observed
            last_error = DeployError(
                "frontend serving identity mismatch: "
                f"expected={expected_version} local={observed['local']} public={observed['public']}"
            )
        except DeployError as exc:
            last_error = exc
        if attempt + 1 < attempts:
            time.sleep(1)
    assert last_error is not None
    raise last_error


def nginx_server_block_bounds(content: str, server_name: str) -> tuple[int, int]:
    """Locate exactly one nginx server block by its exact server_name token."""
    matches: list[tuple[int, int]] = []
    for match in re.finditer(r"(?m)^server\s*\{\s*$", content):
        depth = 0
        end = -1
        for index in range(match.start(), len(content)):
            if content[index] == "{":
                depth += 1
            elif content[index] == "}":
                depth -= 1
                if depth == 0:
                    end = index + 1
                    while end < len(content) and content[end] in " \t":
                        end += 1
                    if end < len(content) and content[end] == "\n":
                        end += 1
                    break
        if end < 0:
            raise DeployError("live test nginx config contains an unterminated server block")
        block = content[match.start():end]
        names = re.findall(r"(?m)^\s*server_name\s+([^;]+);\s*$", block)
        if any(server_name in value.split() for value in names):
            matches.append((match.start(), end))
    if len(matches) != 1:
        raise DeployError(
            f"live test nginx config contains {len(matches)} server blocks for {server_name}"
        )
    return matches[0]


def connected_mail_ingress_block(
    integration_http_port: int, forwarded_proto: str,
) -> str:
    port = str(integration_http_port)
    if not port.isdigit() or not 1 <= int(port) <= 65535:
        raise DeployError("connected-mail Integration port is invalid")
    if forwarded_proto not in {"http", "https"}:
        raise DeployError("connected-mail forwarded protocol is invalid")
    return f"""    # Connected mailbox OAuth and status are Integration-owned in D3. This
    # more-specific prefix must precede the sfcrm /api/ projection below.
    location /api/connected-mailboxes/ {{
        proxy_pass http://127.0.0.1:{port};
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto {forwarded_proto};
    }}
"""


def route_mcp_ingress(rendered: str, context: D3Context) -> str:
    if "mcp_worker" not in context.images:
        return rendered
    for location in ("/mcp", "/.well-known/oauth-protected-resource"):
        pattern = r"(location " + re.escape(location) + r" \{\s*proxy_pass http://127\.0\.0\.1:)" + "(?:" + str(context.ports.sfcrm_http) + "|" + str(context.ports.mcp_http) + ");"
        rendered, count = re.subn(pattern, lambda m: m[1] + str(context.ports.mcp_http) + ";", rendered)
        if count != 1:
            raise DeployError("MCP ingress requires exactly one owned route for " + location)
    return rendered


def render_d3_nginx_server(context: D3Context, frontend: Path) -> str:
    forwarded_proto = (
        urllib.parse.urlparse(context.envv["PUBLIC_BASE_URL"]).scheme or "https"
    )
    rendered = render_template(
        REPOSITORY_ROOT / "deployments/offline/nginx/nexuscrm-env.conf.template",
        {
            "__NGINX_LISTEN_PORT__": context.envv.get("NGINX_LISTEN_PORT", "80"),
            "__NGINX_FORWARDED_PROTO__": forwarded_proto,
            "__SERVER_NAME__": context.envv["SERVER_NAME"],
            "__FRONTEND_DIST__": str(frontend),
            "__BACKEND_PORT__": str(context.ports.sfcrm_http),
            "__CONNECTED_MAIL_INGRESS_BLOCK__": connected_mail_ingress_block(
                context.ports.integration_http, forwarded_proto,
            ),
            "__WEBHOOK_PORT__": str(context.ports.integration_http),
            "__HEALTH_PORT__": str(context.ports.sfcrm_http),
            "__HEALTH_PATH__": "/readyz",
            "__EXP_HUBSPOT_OPTIONS_ADAPTER_PORT__": str(context.ports.integration_http),
            "__ACCESS_LOG__": "/var/log/nginx/nexuscrm-test2.access.log",
            "__ERROR_LOG__": "/var/log/nginx/nexuscrm-test2.error.log",
        },
    )
    rendered = route_mcp_ingress(rendered, context)
    if "__" in rendered:
        raise DeployError("D3 nginx rendering left an unresolved template token")
    return f"{D3_NGINX_MARKER} run={context.run_id}\n{rendered}"


def replace_nginx_server_block(content: str, server_name: str, replacement: str) -> str:
    start, end = nginx_server_block_bounds(content, server_name)
    return content[:start] + replacement.rstrip() + "\n\n" + content[end:].lstrip("\n")


def remove_d3_nginx_markers(content: str) -> str:
    """Remove only deployer-owned marker lines before publishing the next owner."""
    return re.sub(
        rf"(?m)^{re.escape(D3_NGINX_MARKER)}(?: run=[^\s]+)?\s*\n?",
        "",
        content,
    )


def recover_previous_nginx_from_archived_owner(
    context: D3Context,
    state: Mapping[str, Any],
    content: bytes,
) -> dict[str, Any]:
    """Carry the original rollback authority across a serving successor chain."""
    live_digest = sha256_bytes(content)
    candidates: list[tuple[str, str, bytes, str]] = []
    state_root = context.state_file.parent
    for archived in sorted(state_root.glob("superseded-*.json")):
        if archived.is_symlink() or not archived.is_file():
            continue
        archived_state = read_state(archived)
        owner_run_id = archived_state.get("runId")
        if (
            not isinstance(owner_run_id, str)
            or not RUN_ID_RE.fullmatch(owner_run_id)
            or not archived.name.startswith(f"superseded-{owner_run_id}-")
            or archived_state.get("publishedNginxConfigDigest") != live_digest
        ):
            continue
        previous_path = archived_state.get("previousNginxConfig")
        previous_digest = archived_state.get("previousNginxConfigDigest")
        if not isinstance(previous_path, str) or not isinstance(previous_digest, str):
            continue
        source = Path(previous_path)
        if source.is_symlink() or not source.is_file():
            continue
        previous_content = source.read_bytes()
        if sha256_bytes(previous_content) != previous_digest:
            continue
        candidates.append((owner_run_id, previous_digest, previous_content, str(archived)))
    if not candidates:
        raise DeployError("D3 nginx is live without a recoverable rollback authority")
    authorities = {
        (owner_run_id, digest, previous_content)
        for owner_run_id, digest, previous_content, _ in candidates
    }
    if len(authorities) != 1:
        raise DeployError("archived D3 nginx rollback authorities conflict")
    owner_run_id, previous_digest, previous_content = next(iter(authorities))
    backup = context.run_root / "authority" / "nginx-before-d3.conf"
    backup.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
    if backup.is_symlink() or (backup.exists() and not backup.is_file()):
        raise DeployError(f"D3 nginx rollback recovery path is invalid: {backup}")
    if backup.exists() and backup.read_bytes() != previous_content:
        raise DeployError(f"D3 nginx rollback recovery path conflicts: {backup}")
    if not backup.exists():
        pending = backup.with_name(f".{backup.name}.{new_uuid7()}.pending")
        pending.write_bytes(previous_content)
        os.chmod(pending, 0o400)
        os.replace(pending, backup)
    return replace_state(
        context,
        expected_phase=str(state["phase"]),
        updates={
            "previousNginxConfig": str(backup),
            "previousNginxConfigDigest": previous_digest,
            "nginxRollbackRecovery": {
                "ownerRunId": owner_run_id,
                "liveDigest": live_digest,
                "archivedEvidence": sorted({source for _, _, _, source in candidates}),
            },
        },
    )


def capture_previous_nginx(context: D3Context, state: Mapping[str, Any]) -> dict[str, Any]:
    backup_value = state.get("previousNginxConfig")
    digest_value = state.get("previousNginxConfigDigest")
    if isinstance(backup_value, str) and isinstance(digest_value, str):
        backup = Path(backup_value)
        if backup.is_symlink() or not backup.is_file():
            raise DeployError("D3 lost its previous nginx rollback authority")
        if sha256_bytes(backup.read_bytes()) != digest_value:
            raise DeployError("D3 previous nginx rollback authority changed")
        return dict(state)
    if backup_value is not None or digest_value is not None:
        raise DeployError("D3 previous nginx rollback authority is incomplete")
    if TEST2_NGINX_CONF.is_symlink() or not TEST2_NGINX_CONF.is_file():
        raise DeployError(f"live test nginx authority is unavailable: {TEST2_NGINX_CONF}")
    content = TEST2_NGINX_CONF.read_bytes()
    if D3_NGINX_MARKER.encode() in content:
        return recover_previous_nginx_from_archived_owner(context, state, content)
    backup = context.run_root / "authority" / "nginx-before-d3.conf"
    backup.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
    if backup.exists() or backup.is_symlink():
        raise DeployError(f"stale D3 nginx rollback authority exists: {backup}")
    backup.write_bytes(content)
    os.chmod(backup, 0o400)
    return replace_state(
        context, expected_phase=str(state["phase"]),
        updates={
            "previousNginxConfig": str(backup),
            "previousNginxConfigDigest": sha256_bytes(content),
        },
    )


def ensure_d3_public_ingress(context: D3Context) -> dict[str, Any]:
    """Converge frontend and nginx owners for both a fresh and resumed D3 run."""
    state = require_state(
        context, expected_phase={"serving-start-in-progress", "serving"},
    )
    source = Path(str(state["prepared"]["frontendDist"])).resolve()
    frontend, frontend_digest = publish_readable_frontend(context, source)
    state = capture_previous_nginx(context, state)
    current = remove_d3_nginx_markers(TEST2_NGINX_CONF.read_text())
    rendered = replace_nginx_server_block(
        current, context.envv["SERVER_NAME"], render_d3_nginx_server(context, frontend),
    )
    rendered_digest = sha256_bytes(rendered.encode())
    if sha256_bytes(TEST2_NGINX_CONF.read_bytes()) != rendered_digest:
        activate_nginx_conf(
            TEST2_NGINX_CONF,
            rendered,
            healthcheck_url=(
                f"http://127.0.0.1:{context.envv.get('NGINX_LISTEN_PORT', '80')}/health"
            ),
            healthcheck_headers={"Host": context.envv["SERVER_NAME"].split()[0]},
        )
    if sha256_bytes(TEST2_NGINX_CONF.read_bytes()) != rendered_digest:
        raise DeployError("live test2 nginx did not adopt the exact D3 ingress")
    return replace_state(
        context, expected_phase=str(state["phase"]),
        updates={
            "publishedFrontendTarget": str(frontend),
            "publishedFrontendDigest": frontend_digest,
            "publishedNginxConfigDigest": rendered_digest,
        },
    )


def restore_previous_nginx(context: D3Context, state: Mapping[str, Any]) -> None:
    backup_value = state.get("previousNginxConfig")
    digest_value = state.get("previousNginxConfigDigest")
    if backup_value is None and digest_value is None:
        return
    if not isinstance(backup_value, str) or not isinstance(digest_value, str):
        raise DeployError("D3 previous nginx rollback authority is incomplete")
    backup = Path(backup_value)
    if backup.is_symlink() or not backup.is_file() or sha256_bytes(backup.read_bytes()) != digest_value:
        raise DeployError("D3 previous nginx rollback authority changed")
    current_digest = sha256_bytes(TEST2_NGINX_CONF.read_bytes())
    published_digest = state.get("publishedNginxConfigDigest")
    if current_digest not in {digest_value, published_digest}:
        raise DeployError("rollback refuses to overwrite an unknown live nginx owner")
    if current_digest != digest_value:
        activate_nginx_conf(TEST2_NGINX_CONF, backup.read_text())


def require_exact_checkout(
    path: Path,
    expected_commit: str,
    label: str,
    *,
    require_full_history: bool = False,
) -> Path:
    path = path.expanduser().resolve()
    if not (path / ".git").exists():
        raise DeployError(f"exact {label} checkout is unavailable: {path}")
    actual = default_capture(("git", "-C", str(path), "rev-parse", "HEAD"))
    if actual != expected_commit:
        raise DeployError(
            f"exact {label} checkout is at {actual}, resolved BOM requires {expected_commit}"
        )
    if default_capture(("git", "-C", str(path), "status", "--porcelain")):
        raise DeployError(f"exact {label} checkout is dirty: {path}")
    if require_full_history and default_capture((
        "git", "-C", str(path), "rev-parse", "--is-shallow-repository",
    )) != "false":
        raise DeployError(
            f"exact {label} checkout is shallow; product metadata locks require full history"
        )
    return path


def copy_from_image(
    context: D3Context,
    *,
    role: str,
    source: str,
    destination: Path,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> None:
    name = container_name(context.run_id, f"extract-{role}")
    execute((
        "docker", "create", "--name", name,
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        context.images[role], "/__nexuscrm_d3_extract__",
    ))
    try:
        execute(("docker", "cp", f"{name}:{source}", str(destination)))
    finally:
        execute(("docker", "rm", "-f", name))


def copy_from_exact_image(
    context: D3Context,
    *,
    image: str,
    role: str,
    source: str,
    destination: Path,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> None:
    name = container_name(context.run_id, f"extract-{role}")
    execute((
        "docker", "create", "--name", name,
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        image, "/__nexuscrm_d3_extract__",
    ))
    try:
        execute(("docker", "cp", f"{name}:{source}", str(destination)))
    finally:
        execute(("docker", "rm", "-f", name))


def validate_candidate_source_artifacts(
    context: D3Context,
    *,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
    execute: Callable[[Sequence[str]], None],
    l2_checkout_commit: str = "",
) -> tuple[Path, Path, Path, Path, str]:
    """Validate immutable candidate sources without changing serving state."""
    app_packs_root = require_exact_checkout(
        app_packs_root,
        context.bom["app_installation"]["artifact"]["commit"],
        "AppRelease",
        require_full_history=True,
    )
    core_root = require_exact_checkout(
        core_root,
        context.bom["services"]["runtime_api"]["source_commit"],
        "Core",
    )
    effective_l2_commit = l2_checkout_commit or context.bom["l2_metadata"]["commit"]
    l2_root = require_exact_checkout(l2_root, effective_l2_commit, "L2 metadata")
    artifact_errors = verify_app_packs_artifacts(dict(context.bom), app_packs_root)
    if artifact_errors:
        raise DeployError(
            "exact AppRelease verification failed: " + "; ".join(artifact_errors)
        )
    deployment_tools_source = context.resolved_bom.parent / "deployment-tools"
    try:
        verify_deployment_tools(
            root=deployment_tools_source,
            resolved_bom=context.resolved_bom,
        )
    except DeploymentToolError as exc:
        raise DeployError(
            f"exact D3 deployment-tools bundle is invalid: {exc}"
        ) from exc
    approval_publication_source, approval_publication_digest = (
        validate_candidate_approval_publication(context)
    )
    metadata_verifier = (
        REPOSITORY_ROOT / "scripts" / "verify_serviceification_d1_metadata_authority.py"
    )
    execute((
        "python3", str(metadata_verifier), "--bom", str(context.resolved_bom),
        "--l2-root", str(l2_root),
        "--app-packs-root", str(app_packs_root),
    ))
    return (
        app_packs_root,
        core_root,
        l2_root,
        approval_publication_source,
        approval_publication_digest,
    )


def validate_candidate_runtime_compatibility(
    context: D3Context,
    *,
    app_packs_root: Path,
    runtime_manifest: Path,
    catalog_manifest: Path,
) -> None:
    artifact = context.bom["app_installation"]["artifact"]
    package_path = (app_packs_root / str(artifact["path"])).resolve()
    try:
        package_path.relative_to(app_packs_root.resolve())
    except ValueError as exc:
        raise DeployError("AppPackage path escapes its exact checkout") from exc
    package = load_json_object(package_path, "immutable AppPackage")
    try:
        compatibility = host_compatibility(runtime_manifest, catalog_manifest)
        validate_app_engine_compatibility(package, compatibility)
    except ValidationFailure as exc:
        raise DeployError(f"L1 Application is incompatible with L0 Engine: {exc}") from exc


def verify_local_frontend_artifact(
    context: D3Context,
    frontend_dir: Path,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> None:
    if not is_local_test3_bom(context.bom):
        return
    require_local_image_context(context)
    with tempfile.TemporaryDirectory(prefix="nexuscrm-test3-local-frontend-") as raw:
        tuple_path = Path(raw) / "source-tuple.json"
        copy_from_image(
            context, role="frontend", source="/usr/share/nexuscrm/release/source-tuple.json",
            destination=tuple_path, execute=execute,
        )
        if tuple_path.is_symlink() or not tuple_path.is_file():
            raise DeployError("local frontend embeds no regular source tuple")
        source_tuple = load_bom(tuple_path)
        expected = context.bom["frontend"]
        version = load_bom(frontend_dir / "version.json")
        if (
            source_tuple != expected["source_tuple"]
            or canonical_json_sha256(source_tuple) != expected["publication"]["source_tuple_sha256"]
            or version.get("version") != expected["publication"]["source_tuple_sha256"]
        ):
            raise DeployError("local frontend embedded source tuple/version disagrees with selection")


def preflight_candidate_artifacts(
    context: D3Context,
    *,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> None:
    """Prove a successor candidate before stopping the current serving tuple."""
    app_packs_root, _, _, _, _ = validate_candidate_source_artifacts(
        context,
        app_packs_root=app_packs_root,
        core_root=core_root,
        l2_root=l2_root,
        execute=execute,
    )
    pull_and_verify_images(
        context,
        tuple(image_source_commits(context.bom)),
        execute=execute,
        capture_output=capture_output,
    )
    with tempfile.TemporaryDirectory(prefix="nexuscrm-d3-candidate-preflight-") as raw:
        root = Path(raw)
        runtime_manifest = root / "runtime-host-compatibility.json"
        catalog_manifest = root / "catalog-host-compatibility.json"
        frontend_dir = root / "frontend"
        copy_from_exact_image(
            context,
            image=context.images["runtime_api"],
            role="preflight-runtime",
            source="/etc/nexuscrm/host-compatibility.json",
            destination=runtime_manifest,
            execute=execute,
        )
        copy_from_exact_image(
            context,
            image=context.images["catalog_reconcile"],
            role="preflight-catalog",
            source="/etc/nexuscrm/host-compatibility.json",
            destination=catalog_manifest,
            execute=execute,
        )
        copy_from_exact_image(
            context,
            image=context.images["frontend"],
            role="preflight-frontend",
            source="/usr/share/nexuscrm/frontend/.",
            destination=frontend_dir,
            execute=execute,
        )
        validate_candidate_runtime_compatibility(
            context,
            app_packs_root=app_packs_root,
            runtime_manifest=runtime_manifest,
            catalog_manifest=catalog_manifest,
        )
        verify_frontend_dist(frontend_dir)
        verify_local_frontend_artifact(context, frontend_dir, execute=execute)


def prepare_artifacts(
    context: D3Context,
    *,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    l2_checkout_commit: str = "",
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={
            "captured", "catalog-successor-captured", "runtime-successor-captured",
        },
    )
    catalog_successor = state["phase"] == "catalog-successor-captured"
    runtime_successor = state["phase"] == "runtime-successor-captured"
    if owned_containers():
        raise DeployError("D3-owned containers already exist before artifact preparation")
    (
        app_packs_root,
        core_root,
        l2_root,
        approval_publication_source,
        approval_publication_digest,
    ) = validate_candidate_source_artifacts(
        context,
        app_packs_root=app_packs_root,
        core_root=core_root,
        l2_root=l2_root,
        execute=execute,
        l2_checkout_commit=l2_checkout_commit,
    )
    core_commit = context.bom["services"]["runtime_api"]["source_commit"]
    effective_l2_commit = l2_checkout_commit or context.bom["l2_metadata"]["commit"]
    deployment_tools_source = context.resolved_bom.parent / "deployment-tools"
    source_commit_by_role = image_source_commits(context.bom)
    pull_and_verify_images(
        context, tuple(source_commit_by_role),
        execute=execute, capture_output=capture_output,
    )

    authority_dir = context.run_root / "authority"
    frontend_dir = context.run_root / "frontend"
    for path in (authority_dir, frontend_dir):
        if path.is_symlink():
            raise DeployError(f"incomplete D3 artifact path became a symlink: {path}")
        if path.exists():
            shutil.rmtree(path)
    authority_dir.mkdir(mode=0o700)
    frontend_dir.mkdir(mode=0o700)
    copy_from_image(
        context, role="runtime_api", source="/etc/nexuscrm/host-compatibility.json",
        destination=authority_dir / "runtime-host-compatibility.json", execute=execute,
    )
    copy_from_image(
        context, role="catalog_reconcile", source="/etc/nexuscrm/host-compatibility.json",
        destination=authority_dir / "catalog-host-compatibility.json", execute=execute,
    )
    validate_candidate_runtime_compatibility(
        context,
        app_packs_root=app_packs_root,
        runtime_manifest=authority_dir / "runtime-host-compatibility.json",
        catalog_manifest=authority_dir / "catalog-host-compatibility.json",
    )
    copy_from_image(
        context, role="frontend", source="/usr/share/nexuscrm/frontend/.",
        destination=frontend_dir, execute=execute,
    )
    compatibility_paths = (
        authority_dir / "runtime-host-compatibility.json",
        authority_dir / "catalog-host-compatibility.json",
    )
    for path in compatibility_paths:
        if not path.is_file() or path.is_symlink():
            raise DeployError(f"image compatibility authority is not a regular file: {path}")
        os.chmod(path, 0o400)
    deployment_tools = authority_dir / "deployment-tools"
    shutil.copytree(deployment_tools_source, deployment_tools, symlinks=True)
    try:
        tool_manifest = verify_deployment_tools(
            root=deployment_tools,
            resolved_bom=context.resolved_bom,
        )
    except DeploymentToolError as exc:
        raise DeployError(f"copied D3 deployment-tools bundle is invalid: {exc}") from exc
    setup_binary = deployment_tools / DEPLOYMENT_TOOL_FILES["setup"]
    metadata_binary = deployment_tools / DEPLOYMENT_TOOL_FILES["pack_runtime_metadata"]
    os.chmod(setup_binary, 0o500)
    os.chmod(metadata_binary, 0o500)
    os.chmod(deployment_tools / "manifest.json", 0o400)
    approval_publication_path = authority_dir / "approval-publication.json"
    shutil.copyfile(approval_publication_source, approval_publication_path)
    os.chmod(approval_publication_path, 0o400)
    verify_frontend_dist(frontend_dir)
    verify_local_frontend_artifact(context, frontend_dir, execute=execute)
    copied_bom = context.run_root / "resolved-bom.json"
    if copied_bom.is_symlink():
        raise DeployError("run-owned resolved BOM path became a symlink")
    expected_bom = context.resolved_bom.read_bytes()
    if copied_bom.exists():
        if not copied_bom.is_file() or copied_bom.read_bytes() != expected_bom:
            raise DeployError("run-owned resolved BOM copy changed during preparation")
    else:
        copied_bom.write_bytes(expected_bom)
        os.chmod(copied_bom, 0o400)
    if sha256_bytes(copied_bom.read_bytes()) != context.bom_file_digest:
        raise DeployError("run-owned resolved BOM copy changed during preparation")

    replace_state(
        context,
        expected_phase=str(state["phase"]),
        updates={
            # A pre-operation Catalog successor keeps the already-converged
            # database and rebuilds only run-owned artifacts from the latest
            # experimental component heads. It must not replay schema/metadata
            # apply merely because an L0 image digest changed.
            "phase": (
                "runtime-applied" if catalog_successor else
                "runtime-apply-in-progress" if runtime_successor else
                "artifacts-ready"
            ),
            "exactSources": {
                "appPacksRoot": str(app_packs_root),
                "coreRoot": str(core_root),
                "l2Root": str(l2_root),
                "appPacksCommit": context.bom["app_installation"]["artifact"]["commit"],
                "coreCommit": core_commit,
                "l2Commit": effective_l2_commit,
            },
            "prepared": {
                "runtimeHostCompatibility": str(authority_dir / "runtime-host-compatibility.json"),
                "catalogHostCompatibility": str(authority_dir / "catalog-host-compatibility.json"),
                "setupBinary": str(setup_binary),
                "packRuntimeMetadataBinary": str(metadata_binary),
                "approvalPublication": str(approval_publication_path),
                "approvalPublicationDigest": approval_publication_digest,
                "deploymentToolsManifestDigest": sha256_bytes(
                    (deployment_tools / "manifest.json").read_bytes()
                ),
                "setupCoreRevision": str(tool_manifest["coreRevision"]),
                "frontendDist": str(frontend_dir),
                "resolvedBomCopy": str(copied_bom),
            },
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": (
            "runtime-applied" if catalog_successor else
            "runtime-apply-in-progress" if runtime_successor else
            "artifacts-ready"
        ),
        "runId": context.run_id,
        "resolvedBomFileDigest": context.bom_file_digest,
        "images": dict(context.images),
        "sourceCommits": source_commit_by_role,
        "deploymentTools": tool_manifest,
    }


def validate_candidate_approval_publication(context: D3Context) -> tuple[Path, str]:
    source = context.resolved_bom.parent / "approval-publication.json"
    if source.is_symlink() or not source.is_file():
        raise DeployError("exact D3 Approval publication bundle is unavailable")
    try:
        publication = json.loads(source.read_text())
    except (OSError, json.JSONDecodeError) as exc:
        raise DeployError("exact D3 Approval publication bundle is invalid") from exc
    if (
        not isinstance(publication, dict)
        or publication.get("schema") != "nexuscrm.approval-publication-bundle/v1"
        or publication.get("sourceDigest") != context.bom["l2_metadata"]["tree_sha256"]
        or not isinstance(publication.get("definitions"), list)
        or not publication["definitions"]
    ):
        raise DeployError("exact D3 Approval publication authority is incomplete")
    return source, sha256_bytes(source.read_bytes())


def refresh_incompatible_deployment_tools(context: D3Context) -> dict[str, Any]:
    """Resume a pre-Catalog apply with the exact candidate deployment tools.

    The setup executable is a run-owned deployment input, not a serving image
    and not the Catalog job.  Before BeginOperation, a repaired implementation
    may replace the prepared copy when the resolved BOM and live Catalog/Worker
    boundaries are unchanged.  The replacement remains fully checksummed and
    is recorded in state; this is not a mutable PATH fallback.
    """
    state = require_state(context, expected_phase="runtime-apply-in-progress")
    prepared = state.get("prepared")
    if not isinstance(prepared, dict):
        raise DeployError("D3 Runtime apply has no prepared deployment tools")
    old_setup = Path(str(prepared.get("setupBinary", ""))).expanduser()
    if old_setup.is_symlink() or not old_setup.is_file() or not os.access(old_setup, os.X_OK):
        raise DeployError("D3 Runtime apply lost its prepared setup binary")
    old_compatible = True
    try:
        require_setup_commands(old_setup)
    except DeploymentToolError:
        # A binary which does not advertise the invoked command rejects it in
        # command dispatch, before opening the database or running pack logic.
        old_compatible = False

    candidate_root = context.resolved_bom.parent / "deployment-tools"
    try:
        manifest = verify_deployment_tools(
            root=candidate_root,
            resolved_bom=context.resolved_bom,
        )
    except DeploymentToolError as exc:
        if old_compatible:
            return dict(state)
        raise DeployError(f"replacement D3 deployment-tools bundle is invalid: {exc}") from exc
    manifest_digest = sha256_bytes((candidate_root / "manifest.json").read_bytes())
    if old_compatible and manifest_digest == prepared.get("deploymentToolsManifestDigest"):
        return dict(state)

    start = require_catalog_start(state)
    expected_catalog = require_catalog_before(state) if start["mode"] == "UPGRADE" else None
    if read_optional_catalog_authority(context.envv) != expected_catalog:
        raise DeployError("incompatible setup recovery refuses changed Catalog authority")
    worker_table_present, worker_ownership = read_optional_worker_ownership(context.envv)
    expected_worker_ownership = require_worker_ownership_before(state)
    if not worker_table_present or worker_ownership != expected_worker_ownership:
        raise DeployError("incompatible setup recovery refuses changed Worker authority")

    authority_root = context.run_root / "authority"
    target = authority_root / f"deployment-tools-{manifest_digest.removeprefix('sha256:')}"
    if target.exists() or target.is_symlink():
        try:
            verify_deployment_tools(root=target, resolved_bom=context.resolved_bom)
        except DeploymentToolError as exc:
            raise DeployError("existing replacement deployment-tools bundle is invalid") from exc
    else:
        staging_root = authority_root / f".deployment-tools-{new_uuid7()}"
        shutil.copytree(candidate_root, staging_root, symlinks=False)
        try:
            verify_deployment_tools(root=staging_root, resolved_bom=context.resolved_bom)
            os.replace(staging_root, target)
        except BaseException:
            shutil.rmtree(staging_root, ignore_errors=True)
            raise
    setup_binary = target / DEPLOYMENT_TOOL_FILES["setup"]
    metadata_binary = target / DEPLOYMENT_TOOL_FILES["pack_runtime_metadata"]
    os.chmod(setup_binary, 0o500)
    os.chmod(metadata_binary, 0o500)
    os.chmod(target / "manifest.json", 0o400)
    replacement = dict(prepared)
    replacement.update({
        "setupBinary": str(setup_binary),
        "packRuntimeMetadataBinary": str(metadata_binary),
        "deploymentToolsManifestDigest": manifest_digest,
        "setupCoreRevision": str(manifest["coreRevision"]),
    })
    return replace_state(
        context,
        expected_phase="runtime-apply-in-progress",
        updates={
            "prepared": replacement,
            "deploymentToolsRecovery": {
                "reason": (
                    "pre-catalog-deployment-tool-implementation-refresh"
                    if old_compatible else
                    "prior-setup-rejected-required-command-before-dispatch"
                ),
                "supersededManifestDigest": prepared.get("deploymentToolsManifestDigest"),
                "replacementManifestDigest": manifest_digest,
                "setupCoreRevision": manifest["coreRevision"],
            },
        },
    )


def recover_retryable_official_catalog_candidate(
    context: D3Context,
    *,
    retry_app_packs_root: Path,
    retry_app_packs_commit: str,
) -> dict[str, Any]:
    """Finish one exact stopped official-pack attempt before advancing it.

    The recovery source is used only while Runtime is stopped.  It never
    replaces the candidate AppRelease, images, frontend, or serving tuple.
    """
    state = require_state(context, expected_phase="runtime-apply-in-progress")
    if not re.fullmatch(r"[0-9a-f]{40}", retry_app_packs_commit):
        raise DeployError("Catalog retry AppRelease commit is not exact")
    exact_sources = state.get("exactSources")
    prepared = state.get("prepared")
    if not isinstance(exact_sources, dict) or not isinstance(prepared, dict):
        raise DeployError("Catalog retry has no sealed current candidate inputs")
    l2_commit = str(exact_sources.get("l2Commit", ""))
    core_commit = str(exact_sources.get("coreCommit", ""))
    expected_receipt = {
        "retryAppPacksCommit": retry_app_packs_commit,
        "currentAppPacksCommit": str(exact_sources.get("appPacksCommit", "")),
        "currentCoreCommit": core_commit,
        "currentL2Commit": l2_commit,
    }
    existing_receipt = state.get("catalogRetryAppRelease")
    if existing_receipt is not None:
        if (
            not isinstance(existing_receipt, dict)
            or any(existing_receipt.get(key) != value for key, value in expected_receipt.items())
            or not isinstance(existing_receipt.get("packNames"), list)
            or not existing_receipt["packNames"]
        ):
            raise DeployError("Catalog retry receipt disagrees with the requested exact tuple")
        return dict(state)

    retry_app_packs_root = require_exact_checkout(
        retry_app_packs_root,
        retry_app_packs_commit,
        "stopped Catalog retry AppRelease",
    )
    core_root = require_exact_checkout(
        Path(str(exact_sources.get("coreRoot", ""))), core_commit, "Core",
    )
    l2_root = require_exact_checkout(
        Path(str(exact_sources.get("l2Root", ""))), l2_commit, "L2 metadata",
    )
    setup_binary = Path(str(prepared.get("setupBinary", ""))).expanduser()
    metadata_binary = Path(str(prepared.get("packRuntimeMetadataBinary", ""))).expanduser()
    for binary, label in (
        (setup_binary, "setup binary"),
        (metadata_binary, "pack Runtime metadata binary"),
    ):
        if binary.is_symlink() or not binary.is_file() or not os.access(binary, os.X_OK):
            raise DeployError(f"Catalog retry {label} is unavailable or not executable")

    if owned_containers():
        raise DeployError("Catalog retry refuses to run while D3 containers exist")
    staging.quiesce_staging_target_runtime(
        dict(context.envv),
        runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    runtime_env = dict(context.envv)
    runtime_env["NEXUSCRM_OFFICIAL_APP_PACKS_ROOT"] = str(retry_app_packs_root)
    runtime_env["NEXUSCRM_SETUP_BIN"] = str(setup_binary.resolve())
    runtime_env["NEXUSCRM_PACK_RUNTIME_METADATA_BIN"] = str(metadata_binary.resolve())
    runtime_env["DOWNSTREAM_REF"] = l2_commit
    runtime_env["OFFICIAL_APP_PACKS_REF"] = retry_app_packs_commit
    runtime_env["DEPLOY_ID"] = "catalog-retry-" + hashlib.sha256(
        f"{context.run_id}:{retry_app_packs_commit}".encode("utf-8")
    ).hexdigest()[:24]

    lock_path = l2_root / "stack.lock.json"
    lock_data = json.loads(lock_path.read_text())
    if not isinstance(lock_data, dict):
        raise DeployError(f"Catalog retry stack lock is malformed: {lock_path}")
    retryable_names: list[str] = []
    for entry in runtime_deploy_pack_entries(lock_data):
        pack_name, source, pack_dir = runtime_pack_entry_name_and_source(l2_root, entry)
        if pack_dir is not None or not source.startswith("official_repo:"):
            continue
        status, source_kind, source_uri, source_revision = (
            load_installed_pack_convergence_state(runtime_env, pack_name)
        )
        if (
            status in {"installing", "failed"}
            and source_kind == "official_repo"
            and source_uri == pack_name
            and source_revision == retry_app_packs_commit
        ):
            retryable_names.append(pack_name)
    if not retryable_names:
        raise DeployError(
            "Catalog retry found no failed/installing official pack bound to the exact retry commit"
        )

    run_catalog_mutable_source_preconvergence(
        runtime_env, l2_root, core_root, retry_app_packs_root,
    )
    run_catalog_offline_candidate(
        runtime_env, l2_root, core_root, retry_app_packs_root,
    )
    expected_receipt["packNames"] = sorted(retryable_names)
    return replace_state(
        context,
        expected_phase="runtime-apply-in-progress",
        updates={"catalogRetryAppRelease": expected_receipt},
    )


def runtime_l0_schema_convergence_receipt_matches(
    receipt: Any,
    *,
    command: str,
    core_commit: str,
) -> bool:
    """Verify completed schema work without binding it to a later tool repair."""
    return (
        isinstance(receipt, dict)
        and receipt.get("command") == command
        and receipt.get("coreCommit") == core_commit
        and re.fullmatch(
            r"[0-9a-f]{40}", str(receipt.get("setupCoreRevision", ""))
        ) is not None
    )


def catalog_bundle_operation_id(context: D3Context) -> str:
    """Return the retry-stable identity owned only by the application bundle.

    L0 implementation selection is deliberately absent.  The current v10
    carrier has not yet cut over to the standalone App Bundle manifest, so its
    equivalent owner facts are the exact L1 AppRelease/metadata digests and the
    assembled L2 final-pack tree.  Identical bundle facts retry one operation;
    a different bundle starts another without manufacturing a new L1 version.
    """
    app = context.bom.get("app_installation")
    l2 = context.bom.get("l2_metadata")
    if not isinstance(app, dict) or not isinstance(l2, dict):
        raise DeployError("Catalog bundle operation requires application bundle facts")
    facts = {
        "schema": "nexuscrm.catalog-application-bundle-operation/v1",
        "appId": app.get("app_id"),
        "appReleaseDigest": app.get("app_release_digest"),
        "metadataBundleDigest": app.get("metadata_bundle_digest"),
        "l2TreeDigest": l2.get("tree_sha256"),
    }
    if not isinstance(facts["appId"], str) or not facts["appId"]:
        raise DeployError("Catalog bundle operation requires an exact app id")
    for label in ("appReleaseDigest", "metadataBundleDigest", "l2TreeDigest"):
        if not isinstance(facts[label], str) or not re.fullmatch(
            r"sha256:[0-9a-f]{64}", facts[label]
        ):
            raise DeployError(f"Catalog bundle operation requires an exact {label}")
    return "d3-app-bundle-" + hashlib.sha256(canonical_bytes(facts)).hexdigest()


def recover_direct_test_official_pack_source_authority(
    runtime_env: dict[str, str],
    l2_root: Path,
    core_root: Path,
    app_packs_root: Path,
    *,
    catalog_operation_id: str,
) -> dict[str, Any] | None:
    """Return official packs from the retired direct-release alias to their owner."""
    lock_path = l2_root / "stack.lock.json"
    lock_data = load_json_object(lock_path, "D3 stack lock")
    recovered: list[dict[str, str]] = []
    for entry in runtime_deploy_pack_entries(lock_data):
        pack_name, source, pack_dir = runtime_pack_entry_name_and_source(
            l2_root,
            entry,
        )
        if pack_dir is not None or not source.startswith("official_repo:"):
            continue
        status, source_kind, source_uri, source_revision = (
            load_installed_pack_convergence_state(runtime_env, pack_name)
        )
        if not status:
            continue
        if (
            status == "active"
            and source_kind == "official_repo"
            and source_uri == pack_name
        ):
            continue
        direct_release_uri = (
            "urn:nexuscrm:release-pack:pingcap-inc:nexuscrm-app-packs:"
            + urllib.parse.quote(pack_name, safe="")
        )
        if not (
            status == "active"
            and source_kind == "release_repo"
            and source_uri == direct_release_uri
            and re.fullmatch(r"[0-9a-f]{40}", source_revision)
        ):
            raise DeployError(
                "D3 official pack source authority is neither canonical nor the "
                "exact retired direct-Test alias: "
                f"name={pack_name} status={status} kind={source_kind or '<empty>'} "
                f"uri={source_uri or '<empty>'} revision={source_revision or '<empty>'}"
            )
        recovered.append(
            {
                "name": pack_name,
                "fromKind": source_kind,
                "fromURI": source_uri,
                "fromRevision": source_revision,
            }
        )
    if not recovered:
        return None

    recovery_env = dict(runtime_env)
    recovery_env["DEPLOY_ID"] = "d3-source-authority-" + hashlib.sha256(
        catalog_operation_id.encode("utf-8")
    ).hexdigest()[:24]
    names = [item["name"] for item in recovered]
    run_catalog_official_source_set(
        recovery_env,
        core_root,
        app_packs_root,
        names,
    )
    target_revision = str(runtime_env.get("OFFICIAL_APP_PACKS_REF", "")).strip()
    for pack_name in names:
        status, source_kind, source_uri, source_revision = (
            load_installed_pack_convergence_state(runtime_env, pack_name)
        )
        if (
            status != "active"
            or source_kind != "official_repo"
            or source_uri != pack_name
            or source_revision != target_revision
        ):
            raise DeployError(
                "D3 official pack source authority recovery did not converge: "
                f"name={pack_name} status={status or '<empty>'} "
                f"kind={source_kind or '<empty>'} uri={source_uri or '<empty>'} "
                f"revision={source_revision or '<empty>'}"
            )
    return {
        "operationId": recovery_env["DEPLOY_ID"],
        "targetKind": "official_repo",
        "targetRevision": target_revision,
        "packs": recovered,
    }


def apply_d3_release_app_shell(
    context: D3Context,
    runtime_env: dict[str, str],
    l2_root: Path,
    l2_commit: str,
    *, local_controlplane_source_revision: str = "",
) -> dict[str, Any]:
    """Apply the exact L2 app shell with the reviewed D3 controller."""
    if not re.fullmatch(r"[0-9a-f]{40}", l2_commit):
        raise DeployError("D3 app-shell apply requires an exact L2 commit")
    l2_root = require_exact_checkout(l2_root, l2_commit, "L2 metadata")
    config_path = offline.runtime_app_shell_config(l2_root)
    config_sha256 = hashlib.sha256(config_path.read_bytes()).hexdigest()
    navigation_options = {}
    if any(g["capability_id"] == "runtime.application-navigation.v1"
           for g in context.bom["catalog_consumers"]["runtime_capability_grants"]):
        sources = require_state(context)["exactSources"]
        app_root = require_exact_checkout(Path(sources["appPacksRoot"]), sources["appPacksCommit"], "L1 navigation")
        navigation_options = {"navigation_catalog_image": context.images["catalog_reconcile"],
                              "navigation_pages": app_root / "appservices/sfcrm/navigation_pages.json"}
    result = offline.apply_runtime_app_shell(
        runtime_env,
        l2_root,
        controller_root=REPOSITORY_ROOT,
        output_path=context.run_root / "app-shell.json",
        expected_config_sha256=config_sha256,
        controlplane_image=context.images["catalog_reconcile"] if navigation_options else "",
        local_candidate=context.resolved_bom if is_local_test3_bom(context.bom) else None,
        local_controlplane_source_revision=local_controlplane_source_revision,
        **navigation_options,

    )
    expected_scalars = {
        "schema_version": 1,
        "contract": "runtime-app-shell",
        "config_sha256": config_sha256,
        "status": "passed",
        "mode": "apply-and-assert",
    }
    if any(result.get(key) != value for key, value in expected_scalars.items()):
        raise DeployError("D3 app-shell apply returned incompatible release evidence")
    typed_fields = {
        "app_id": str,
        "frontend_app_id": str,
        "is_default": bool,
        "navigation_item_ids": list,
        "navigation_revision": int,
        "active_profile_count": int,
        "default_profile_count": int,
    }
    if any(
        not isinstance(result.get(key), value_type)
        or (value_type is int and isinstance(result.get(key), bool))
        for key, value_type in typed_fields.items()
    ):
        raise DeployError("D3 app-shell apply returned malformed release evidence")
    if (
        not result["app_id"]
        or not result["frontend_app_id"]
        or not all(
            isinstance(item, str) and item
            for item in result["navigation_item_ids"]
        )
        or result["navigation_revision"] < (1 if navigation_options else 0)
        or result["active_profile_count"] < 0
        or result["default_profile_count"] < 0
    ):
        raise DeployError("D3 app-shell apply returned incomplete release evidence")
    receipt = {
        "schema": D3_APP_SHELL_RECEIPT_SCHEMA,
        "status": "passed",
        "l2Commit": l2_commit,
        "configSha256": config_sha256,
        "appId": result["app_id"],
        "frontendAppId": result["frontend_app_id"],
        "isDefault": result["is_default"],
        "navigationItemIds": list(result["navigation_item_ids"]),
        "navigationRevision": result["navigation_revision"],
        "activeProfileCount": result["active_profile_count"],
        "defaultProfileCount": result["default_profile_count"],
        "mode": result["mode"],
    }

    if navigation_options:
        initialization = result.get("navigation_initialization")
        if not isinstance(initialization, dict) or initialization.get("revision", 0) < 1:
            raise DeployError("D3 app shell lacks current navigation initialization evidence")
        receipt["navigationInitialization"] = initialization
    return receipt


def runtime_app_shell_receipt_matches(
    receipt: object,
    *,
    l2_root: Path,
    l2_commit: str,
    navigation_required: bool = False,
) -> bool:
    if not isinstance(receipt, dict):
        return False
    try:
        l2_root = require_exact_checkout(l2_root, l2_commit, "L2 metadata")
        config_path = offline.runtime_app_shell_config(l2_root)
        config_bytes = config_path.read_bytes()
        config_sha256 = hashlib.sha256(config_bytes).hexdigest()
        config = json.loads(config_bytes)
    except (DeployError, OSError, json.JSONDecodeError):
        return False
    if not isinstance(config, dict) or set(receipt) - {"navigationInitialization"} != D3_APP_SHELL_RECEIPT_KEYS:
        return False
    if navigation_required:
        initialization = receipt.get("navigationInitialization")
        if (not isinstance(initialization, dict)
                or not isinstance(initialization.get("revision"), int)
                or initialization["revision"] < 1):
            return False
    app_id = config.get("id")
    frontend_app_id = config.get("frontendAppId")
    is_default = config.get("isDefault")
    navigation_items = config.get("navigationItems")
    if (
        not isinstance(app_id, str)
        or not app_id
        or not isinstance(frontend_app_id, str)
        or not frontend_app_id
        or not isinstance(is_default, bool)
        or not isinstance(navigation_items, list)
        or not all(
            isinstance(item, dict)
            and isinstance(item.get("id"), str)
            and bool(item["id"])
            for item in navigation_items
        )
    ):
        return False
    active_profile_count = receipt.get("activeProfileCount")
    default_profile_count = receipt.get("defaultProfileCount")
    if (
        not isinstance(active_profile_count, int)
        or isinstance(active_profile_count, bool)
        or active_profile_count < 0
        or not isinstance(default_profile_count, int)
        or isinstance(default_profile_count, bool)
        or default_profile_count < 0
        or default_profile_count != (active_profile_count if is_default else 0)
    ):
        return False
    return (
        receipt.get("schema") == D3_APP_SHELL_RECEIPT_SCHEMA
        and receipt.get("status") == "passed"
        and receipt.get("mode") == "apply-and-assert"
        and receipt.get("l2Commit") == l2_commit
        and receipt.get("configSha256") == config_sha256
        and receipt.get("appId") == app_id
        and receipt.get("frontendAppId") == frontend_app_id
        and receipt.get("isDefault") == is_default
        and isinstance(receipt.get("navigationRevision"), int)
        and not isinstance(receipt["navigationRevision"], bool)
        and receipt["navigationRevision"] >= (1 if navigation_required else 0)
        and isinstance(receipt.get("navigationItemIds"), list)
        and all(isinstance(item, str) and item for item in receipt["navigationItemIds"])
        and len(set(receipt["navigationItemIds"])) == len(receipt["navigationItemIds"])
    )


def recover_d3_release_app_shell_before_catalog(
    context: D3Context,
    l2_root: Path,
    l2_commit: str,
    *,
    expected_catalog: CatalogAuthority | None,
    expected_worker_ownership: Mapping[str, WorkerOwnership],
    quiesce_runtime: Callable[..., None] | None = None,
    apply_app_shell: Callable[..., dict[str, Any]] = apply_d3_release_app_shell,
) -> dict[str, Any]:
    """Recover a legacy receipt only after re-establishing single-writer state."""
    if owned_containers():
        raise DeployError("D3 app-shell recovery refuses active D3 containers")
    (quiesce_runtime or staging.quiesce_staging_target_runtime)(
        dict(context.envv),
        runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    if owned_containers():
        raise DeployError("D3 app-shell recovery did not quiesce D3 containers")
    if read_optional_catalog_authority(context.envv) != expected_catalog:
        raise DeployError("Catalog authority changed during D3 app-shell recovery")
    worker_table_present, worker_ownership = read_optional_worker_ownership(
        context.envv
    )
    if not worker_table_present or worker_ownership != expected_worker_ownership:
        raise DeployError(
            "Runtime Worker ownership changed during D3 app-shell recovery"
        )
    app_shell_receipt = apply_app_shell(
        context,
        dict(context.envv),
        l2_root,
        l2_commit,
    )
    return replace_state(
        context,
        expected_phase="runtime-applied",
        updates={"runtimeAppShell": app_shell_receipt},
    )


def stopped_upgrade_runtime_env(
    envv: Mapping[str, str],
    *,
    l2_root: Path,
    app_packs_root: Path,
    exact_sources: Mapping[str, Any],
    prepared_binaries: Mapping[str, Path],
    catalog_operation_id: str,
) -> dict[str, str]:
    """Bind setup and post-activation recovery to the same candidate sources."""
    return {
        **envv,
        "NEXUSCRM_OFFICIAL_APP_PACKS_ROOT": str(app_packs_root),
        "NEXUSCRM_SETUP_BIN": str(prepared_binaries["setupBinary"]),
        "NEXUSCRM_PACK_RUNTIME_METADATA_BIN": str(
            prepared_binaries["packRuntimeMetadataBinary"]
        ),
        "DOWNSTREAM_REF": str(exact_sources["l2Commit"]),
        # Do not inherit the previous deployment's recovery checkout.
        "NEXUSCRM_RECOVERY_RELEASE_PACK_ROOT": str(l2_root),
        "OFFICIAL_APP_PACKS_REF": str(exact_sources["appPacksCommit"]),
        "DEPLOY_ID": catalog_operation_id,
    }


def apply_stopped_upgrade_runtime(
    context: D3Context,
    *,
    apply_runtime: Callable[..., None] = staging.run_catalog_offline_candidate_phase,
    apply_app_shell: Callable[..., dict[str, Any]] = apply_d3_release_app_shell,
    quiesce_runtime: Callable[..., None] | None = None,
    catalog_retry_app_packs_root: Path | None = None,
    catalog_retry_app_packs_commit: str | None = None,
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={
            "quiesced", "runtime-apply-in-progress", "runtime-candidate-applied",
        },
    )
    if not isinstance(state.get("baselineReset"), dict):
        raise DeployError("D3 stopped upgrade has no production-baseline reset receipt")
    start = require_catalog_start(state)
    before = require_catalog_before(state) if start["mode"] == "UPGRADE" else None
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, dict):
        raise DeployError("D3 stopped upgrade has no exact source tuple")
    core_root = Path(str(exact_sources.get("coreRoot", ""))).resolve()
    app_packs_root = Path(str(exact_sources.get("appPacksRoot", ""))).resolve()
    l2_root = Path(str(exact_sources.get("l2Root", ""))).resolve()
    for path, label in (
        (core_root, "Core"), (app_packs_root, "AppRelease"), (l2_root, "L2 metadata"),
    ):
        if not path.is_dir():
            raise DeployError(f"exact {label} checkout disappeared before setup apply")
    prepared = state.get("prepared")
    if not isinstance(prepared, dict):
        raise DeployError("D3 stopped upgrade has no prepared artifact tuple")
    prepared_binaries: dict[str, Path] = {}
    for key, label in (
        ("setupBinary", "setup binary"),
        ("packRuntimeMetadataBinary", "pack runtime metadata binary"),
    ):
        raw_path = Path(str(prepared.get(key, ""))).expanduser()
        if raw_path.is_symlink() or not raw_path.is_file() or not os.access(raw_path, os.X_OK):
            raise DeployError(f"prepared D3 {label} is unavailable or not executable")
        prepared_binaries[key] = raw_path.resolve()
    if owned_containers():
        raise DeployError("stopped Runtime apply refuses to run while D3 containers exist")
    # A retry may follow an interrupted recovery attempt that tried to restart
    # the legacy units. Re-establish the single-writer boundary every time,
    # immediately before any InstalledPack or schema mutation.
    (quiesce_runtime or staging.quiesce_staging_target_runtime)(
        dict(context.envv),
        runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    if state["phase"] == "quiesced":
        current = read_optional_catalog_authority(context.envv)
        if current != before:
            raise DeployError("Catalog authority changed after D3 quiescence")
        state = replace_state(
            context,
            expected_phase="quiesced",
            updates={"phase": "runtime-apply-in-progress"},
        )
    catalog_operation_id = catalog_bundle_operation_id(context)
    runtime_env = stopped_upgrade_runtime_env(
        context.envv,
        l2_root=l2_root,
        app_packs_root=app_packs_root,
        exact_sources=exact_sources,
        prepared_binaries=prepared_binaries,
        catalog_operation_id=catalog_operation_id,
    )
    candidate_already_applied = state["phase"] == "runtime-candidate-applied"

    schema_receipt = state.get("runtimeL0SchemaConvergence")
    expected_schema_receipt = {
        "command": "ensure-l0-schema-ddl",
        "coreCommit": exact_sources["coreCommit"],
        "setupCoreRevision": prepared.get("setupCoreRevision"),
    }
    if schema_receipt is None:
        # A stopped L0 release can add neutral system tables independently of
        # L1/L2 pack metadata. Converge that additive physical schema before
        # source-pack mutation; do not route it through a full ApplyAll.
        run_backend_setup(
            runtime_env,
            core_root,
            app_packs_root,
            ["ensure-l0-schema-ddl"],
        )
        state = replace_state(
            context,
            expected_phase=str(state["phase"]),
            updates={"runtimeL0SchemaConvergence": expected_schema_receipt},
        )
    elif not runtime_l0_schema_convergence_receipt_matches(
        schema_receipt,
        command=expected_schema_receipt["command"],
        core_commit=expected_schema_receipt["coreCommit"],
    ):
        raise DeployError("stopped Runtime apply has a mixed L0 schema convergence receipt")

    if catalog_retry_app_packs_root is not None:
        if not catalog_retry_app_packs_commit:
            raise DeployError("Catalog retry root requires its exact AppRelease commit")
        recover_retryable_official_catalog_candidate(
            context,
            retry_app_packs_root=catalog_retry_app_packs_root,
            retry_app_packs_commit=catalog_retry_app_packs_commit,
        )

    def checkpoint_candidate_applied() -> None:
        replace_state(
            context,
            expected_phase="runtime-apply-in-progress",
            updates={
                "phase": "runtime-candidate-applied",
                "runtimeCandidateApply": {
                    "profile": "official-staging-warm-apply",
                    "coreCommit": exact_sources["coreCommit"],
                    "setupCoreRevision": prepared.get("setupCoreRevision"),
                    "appPacksCommit": exact_sources["appPacksCommit"],
                    "l2Commit": exact_sources["l2Commit"],
                    "catalogOperationKind": start["mode"],
                    "catalogOperationId": catalog_operation_id,
                },
            },
        )

    def recover_direct_test_source_authority() -> None:
        recovery = recover_direct_test_official_pack_source_authority(
            runtime_env,
            l2_root,
            core_root,
            app_packs_root,
            catalog_operation_id=catalog_operation_id,
        )
        if recovery is not None:
            replace_state(
                context,
                expected_phase="runtime-apply-in-progress",
                updates={"directTestSourceAuthorityRecovery": recovery},
            )

    apply_runtime(
        runtime_env,
        l2_root,
        core_root,
        app_packs_root,
        candidate_already_applied=candidate_already_applied,
        pre_candidate_recovery=(
            None if candidate_already_applied else recover_direct_test_source_authority
        ),
        on_candidate_applied=(
            None if candidate_already_applied else checkpoint_candidate_applied
        ),
    )
    require_state(context, expected_phase="runtime-candidate-applied")
    app_shell_receipt = apply_app_shell(
        context,
        runtime_env,
        l2_root,
        str(exact_sources["l2Commit"]),
    )
    inspection_fixtures = apply_test2_inspection_catalog_fixtures(
        context.envv,
        l2_root,
    )
    if read_optional_catalog_authority(context.envv) != before:
        raise DeployError("official setup apply changed immutable Catalog publication authority")
    worker_table_present, worker_ownership = read_optional_worker_ownership(context.envv)
    if not worker_table_present:
        raise DeployError("official setup apply did not create Runtime Worker ownership authority")
    if worker_ownership != require_worker_ownership_before(state):
        raise DeployError("official setup apply changed Runtime Worker ownership authority")
    replace_state(
        context,
        expected_phase="runtime-candidate-applied",
        updates={
            "phase": "runtime-applied",
            "runtimeApply": {
                "profile": "official-staging-warm-apply",
                "coreCommit": exact_sources["coreCommit"],
                "setupCoreRevision": prepared.get("setupCoreRevision"),
                "appPacksCommit": exact_sources["appPacksCommit"],
                "l2Commit": exact_sources["l2Commit"],
                "catalogOperationKind": start["mode"],
                "catalogOperationId": catalog_operation_id,
            },
            "runtimeAppShell": app_shell_receipt,
            "test2InspectionCatalogFixtures": inspection_fixtures,
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "runtime-applied",
        "runId": context.run_id,
        "profile": "official-staging-warm-apply",
        "sourceCommits": {
            "core": exact_sources["coreCommit"],
            "appPacks": exact_sources["appPacksCommit"],
            "l2": exact_sources["l2Commit"],
        },
        "appShell": app_shell_receipt,
    }


def apply_test2_inspection_catalog_fixtures(
    envv: dict[str, str],
    l2_root: Path,
    *,
    execute: Callable[..., None] = db_exec,
    query: Callable[..., str] = db_query_scalar,
) -> dict[str, Any]:
    """Converge shared D3 inspection metadata before Catalog seals a revision."""
    relative_path = Path(
        "deployments/offline/runtime/fixtures/test2-inspection/catalog-authority.sql"
    )
    fixture_path = (l2_root / relative_path).resolve()
    if (
        not fixture_path.is_file()
        or fixture_path.is_symlink()
        or l2_root.resolve() not in fixture_path.parents
    ):
        raise DeployError(f"missing regular Test2 inspection fixture: {relative_path}")
    database = str(envv.get("TIDB_DATABASE", "")).strip()
    if not database:
        raise DeployError("Test2 inspection fixture requires TIDB_DATABASE")

    fixture_bytes = fixture_path.read_bytes()
    task_whoid_state = query(
        envv,
        """
SELECT CONCAT(
  (SELECT COUNT(*)
     FROM `Group`
    WHERE `Id` = CONCAT('00G', SUBSTRING(MD5('test2-inspection:task-whoid:group'), 1, 15))
       OR `DeveloperName` = 'Test2_Inspection_Task_WhoId_Group'),
  '\t',
  (SELECT COUNT(*)
     FROM `Group`
    WHERE `Id` = CONCAT('00G', SUBSTRING(MD5('test2-inspection:task-whoid:group'), 1, 15))
      AND `Name` = 'Test2 Inspection Task WhoId Group'
      AND `DeveloperName` = 'Test2_Inspection_Task_WhoId_Group'
      AND `Label` = 'Test2 Inspection Task WhoId Group'
      AND `Type` = 'Regular'
      AND COALESCE(`IsDeleted`, 0) = 0),
  '\t',
  (SELECT COUNT(*)
     FROM `SharingRule`
    WHERE `Id` = CONCAT('01q', SUBSTRING(MD5('test2-inspection:task-whoid:lead-rule'), 1, 15))
       OR `Name` = 'Test2InspectionTaskWhoIdLeadRule'),
  '\t',
  (SELECT COUNT(*)
     FROM `SharingRule`
    WHERE `Id` = CONCAT('01q', SUBSTRING(MD5('test2-inspection:task-whoid:lead-rule'), 1, 15))
      AND `ObjectApiName` = 'Lead'
      AND `Name` = 'Test2InspectionTaskWhoIdLeadRule'
      AND `Criteria` = 'Company == \"Test2 Inspection Task WhoId Fixture\"'
      AND `AccessLevel` = 'Read'
      AND `ShareWithRoleId` IS NULL
      AND `ShareWithGroupId` = CONCAT('00G', SUBSTRING(MD5('test2-inspection:task-whoid:group'), 1, 15))
      AND `SharedFromRoleId` IS NULL
      AND `SharedFromGroupId` IS NULL
      AND COALESCE(`IsDeleted`, 0) = 0)
);
""".strip(),
        database=database,
    ).strip()
    task_whoid_state_parts = task_whoid_state.split("\t")
    if len(task_whoid_state_parts) != 4 or any(
        value not in {"0", "1"} for value in task_whoid_state_parts
    ):
        raise DeployError(
            "Test2 Task WhoId Catalog fixture found ambiguous retained authority: "
            f"state={task_whoid_state!r}"
        )
    group_rows, exact_group_rows, rule_rows, exact_rule_rows = task_whoid_state_parts
    if (group_rows == "1") != (exact_group_rows == "1"):
        raise DeployError(
            "Test2 Task WhoId Catalog fixture found a conflicting Group identity; "
            "refusing to overwrite or reactivate it"
        )
    if (rule_rows == "1") != (exact_rule_rows == "1"):
        raise DeployError(
            "Test2 Task WhoId Catalog fixture found a conflicting SharingRule identity; "
            "refusing to overwrite or reactivate it"
        )
    source_role_rows = query(
        envv,
        """
SELECT COUNT(*)
FROM `UserRole`
WHERE `Name` = 'NA Sales'
  AND COALESCE(`IsDeleted`, 0) = 0;
""".strip(),
        database=database,
    ).strip()
    target_role_rows = query(
        envv,
        """
SELECT COUNT(*)
FROM `UserRole`
WHERE `Name` = 'APAC Sales Team B'
  AND COALESCE(`IsDeleted`, 0) = 0;
""".strip(),
        database=database,
    ).strip()
    existing_rule_state = query(
        envv,
        """
SELECT CONCAT(
  (SELECT COUNT(*)
     FROM `SharingRule`
    WHERE `Name` = 'StagingInspectionLeadTransferShareRule'),
  '\t',
  (SELECT COUNT(*)
     FROM `SharingRule` sr
     JOIN `UserRole` source_role ON source_role.`Id` = sr.`SharedFromRoleId`
     JOIN `UserRole` target_role ON target_role.`Id` = sr.`ShareWithRoleId`
    WHERE sr.`ObjectApiName` = 'Lead'
      AND sr.`Name` = 'StagingInspectionLeadTransferShareRule'
      AND sr.`Criteria` IS NULL
      AND sr.`AccessLevel` = 'Read'
      AND sr.`SharedFromGroupId` IS NULL
      AND sr.`ShareWithGroupId` IS NULL
      AND COALESCE(sr.`IsDeleted`, 0) = 0
      AND source_role.`Name` = 'NA Sales'
      AND COALESCE(source_role.`IsDeleted`, 0) = 0
      AND target_role.`Name` = 'APAC Sales Team B'
      AND COALESCE(target_role.`IsDeleted`, 0) = 0),
  '\t',
  (SELECT COUNT(*)
     FROM `SharingRule`
    WHERE `Id` = CONCAT('01q', SUBSTRING(MD5('test2-inspection:lead-transfer:owner-rule'), 1, 15))
      AND `Name` <> 'StagingInspectionLeadTransferShareRule')
);
""".strip(),
        database=database,
    ).strip()
    if source_role_rows != "1" or target_role_rows != "1":
        raise DeployError(
            "Test2 Lead transfer Catalog fixture requires one exact source and target role: "
            f"source_count={source_role_rows!r} target_count={target_role_rows!r}"
        )
    rule_state_parts = existing_rule_state.split("\t")
    if len(rule_state_parts) != 3 or any(
        value not in {"0", "1"} for value in rule_state_parts
    ):
        raise DeployError(
            "Test2 Lead transfer Catalog fixture found ambiguous retained rules: "
            f"state={existing_rule_state!r}"
        )
    named_rule_rows, exact_rule_rows, deterministic_id_conflicts = rule_state_parts
    if deterministic_id_conflicts != "0":
        raise DeployError(
            "Test2 Lead transfer Catalog fixture deterministic rule id is already owned"
        )
    if (named_rule_rows == "1") != (exact_rule_rows == "1"):
        raise DeployError(
            "Test2 Lead transfer Catalog fixture found a conflicting retained rule; "
            "refusing to replace or reactivate it"
        )
    execute(envv, fixture_bytes.decode("utf-8"), database=database)
    exact_rows = query(
        envv,
        """
SELECT COUNT(*)
FROM `SharingRule` sr
JOIN `Group` g ON g.`Id` = sr.`ShareWithGroupId`
WHERE sr.`ObjectApiName` = 'Lead'
  AND sr.`Name` = 'Test2InspectionTaskWhoIdLeadRule'
  AND sr.`Criteria` = 'Company == \"Test2 Inspection Task WhoId Fixture\"'
  AND sr.`AccessLevel` = 'Read'
  AND sr.`ShareWithRoleId` IS NULL
  AND sr.`SharedFromRoleId` IS NULL
  AND sr.`SharedFromGroupId` IS NULL
  AND COALESCE(sr.`IsDeleted`, 0) = 0
  AND g.`DeveloperName` = 'Test2_Inspection_Task_WhoId_Group'
  AND g.`Type` = 'Regular'
  AND COALESCE(g.`IsDeleted`, 0) = 0;
""".strip(),
        database=database,
    ).strip()
    if exact_rows != "1":
        raise DeployError(
            "Test2 Task WhoId Catalog fixture did not converge to exactly one rule/group pair: "
            f"count={exact_rows!r}"
        )
    lead_transfer_rows = query(
        envv,
        """
SELECT COUNT(*)
FROM `SharingRule` sr
JOIN `UserRole` source_role ON source_role.`Id` = sr.`SharedFromRoleId`
JOIN `UserRole` target_role ON target_role.`Id` = sr.`ShareWithRoleId`
WHERE sr.`ObjectApiName` = 'Lead'
  AND sr.`Name` = 'StagingInspectionLeadTransferShareRule'
  AND sr.`Criteria` IS NULL
  AND sr.`AccessLevel` = 'Read'
  AND sr.`SharedFromGroupId` IS NULL
  AND sr.`ShareWithGroupId` IS NULL
  AND COALESCE(sr.`IsDeleted`, 0) = 0
  AND source_role.`Name` = 'NA Sales'
  AND COALESCE(source_role.`IsDeleted`, 0) = 0
  AND target_role.`Name` = 'APAC Sales Team B'
  AND COALESCE(target_role.`IsDeleted`, 0) = 0;
""".strip(),
        database=database,
    ).strip()
    if lead_transfer_rows != "1":
        raise DeployError(
            "Test2 Lead transfer Catalog fixture did not converge to exactly one role rule: "
            f"count={lead_transfer_rows!r}"
        )
    return {
        "schema": "nexuscrm.test2-inspection-catalog-fixtures/v2",
        "sourcePath": str(relative_path),
        "sourceDigest": sha256_bytes(fixture_bytes),
        "taskWhoIdRuleCount": 1,
        "leadTransferRuleCount": 1,
    }


def owned_containers(
    *,
    inspect: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
) -> list[str]:
    result = inspect(
        [
            "docker", "ps", "-a", "--filter",
            f"label={CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
            "--format", "{{.Names}}",
        ],
        check=True,
        capture_output=True,
        text=True,
    )
    return sorted(name for name in result.stdout.splitlines() if name)


def preflight(
    context: D3Context,
    *,
    allow_existing_state: bool = False,
) -> dict[str, Any]:
    if is_local_test3_bom(context.bom):
        require_local_image_context(context)
    require_commands("docker", "curl", "jq", "openssl", "systemctl")
    if (
        not allow_existing_state
        and (context.state_file.exists() or context.state_file.is_symlink())
    ):
        state = read_state(context.state_file)
        raise DeployError(
            f"D3 test2 already has active state run={state.get('runId')} phase={state.get('phase')}"
        )
    if "gateway" in context.bom["services"] or any(
        "gateway" in image.lower() for image in context.images.values()
    ):
        raise DeployError("D3 tuple contains a forbidden generic Gateway artifact")
    if len(context.bom["runtime_worker_roles"]) != len(context.worker_stems):
        raise DeployError("D3 tuple Worker role authority is inconsistent")
    actual_families = {role.get("work_family") for role in context.bom["runtime_worker_roles"]}
    if set(context.worker_families) != actual_families:
        raise DeployError("D3 Runtime Worker role set is incomplete or contains another owner")
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "preflight",
        **local_deployment_evidence(context.bom),
        "envName": context.envv["ENV_NAME"],
        "runId": context.run_id,
        "resolvedBomFileDigest": context.bom_file_digest,
        "desiredBomDigest": context.bom["desired_bom_sha256"],
        "images": dict(context.images),
        "ports": context.ports.all_ports(),
        "gatewayArtifacts": 0,
        "runtimeEndpoints": 1,
        "workerArtifactCount": 1,
        "workerRoleCount": len(context.worker_stems),
    }


def capture(context: D3Context) -> dict[str, Any]:
    evidence = preflight(context)
    state = state_payload(context)
    write_new_state(context, state)
    evidence["stateFile"] = str(context.state_file)
    evidence["previousSystemdUnits"] = state["previousSystemdUnits"]
    return evidence


def catalog_implementation_only_successor(
    active_bom: Mapping[str, Any], successor_bom: Mapping[str, Any],
) -> bool:
    """Accept only a new implementation of the same one-shot Catalog job."""
    active_jobs = active_bom.get("jobs")
    successor_jobs = successor_bom.get("jobs")
    if not isinstance(active_jobs, dict) or not isinstance(successor_jobs, dict):
        return False
    active_job = active_jobs.get("catalog_reconcile")
    successor_job = successor_jobs.get("catalog_reconcile")
    if not isinstance(active_job, dict) or not isinstance(successor_job, dict):
        return False
    if active_job.get("oci_repository") != successor_job.get("oci_repository"):
        return False
    allowed_job_fields = {
        "digest", "oci_repository", "provenance_digest", "sbom_digest",
        "source_commit", "source_repository",
    }
    if set(active_job) != allowed_job_fields or set(successor_job) != allowed_job_fields:
        return False
    if active_job.get("source_repository") != successor_job.get("source_repository"):
        return False
    if active_job == successor_job:
        return False
    projected = copy.deepcopy(dict(active_bom))
    projected["desired_bom_sha256"] = successor_bom.get("desired_bom_sha256")
    projected["jobs"]["catalog_reconcile"] = copy.deepcopy(successor_job)
    return canonical_bytes(projected) == canonical_bytes(dict(successor_bom))


def compatible_successor_worker_roles(
    base_roles: object,
    successor_roles: object,
    effective_roles: Sequence[Mapping[str, Any]] | None = None,
) -> list[dict[str, Any]] | None:
    """Accept each successor role at either its release or sealed effective policy."""
    if not isinstance(base_roles, list) or not isinstance(successor_roles, list):
        return None
    current_roles: object = list(effective_roles) if effective_roles is not None else base_roles
    if not isinstance(current_roles, list):
        return None

    def indexed(roles: list[object]) -> dict[tuple[object, object], dict[str, Any]] | None:
        result: dict[tuple[object, object], dict[str, Any]] = {}
        for role in roles:
            if not isinstance(role, dict):
                return None
            owner = (role.get("work_family"), role.get("config_schema"))
            if owner in result:
                return None
            result[owner] = role
        return result

    base_by_owner = indexed(base_roles)
    current_by_owner = indexed(current_roles)
    successor_by_owner = indexed(successor_roles)
    if None in (base_by_owner, current_by_owner, successor_by_owner):
        return None
    assert base_by_owner is not None and current_by_owner is not None and successor_by_owner is not None
    if set(current_by_owner) != set(base_by_owner):
        return None
    base_families = {str(owner[0]) for owner in base_by_owner}
    successor_families = {str(owner[0]) for owner in successor_by_owner}
    allowed_sets = READABLE_WORKER_FAMILY_SETS
    if (
        frozenset(base_families) not in allowed_sets
        or frozenset(successor_families) not in allowed_sets
        or not base_families.issubset(successor_families)
    ):
        return None
    added = set(successor_by_owner) - set(base_by_owner)
    if not added.issubset({
        ("runtime.bulk-ingest.v1", "nexuscrm.runtime-worker-role/v3"),
        ("runtime.permission-authoring-command-lifecycle.v1", "nexuscrm.runtime-worker-role/v3"),
    }):
        return None
    for owner, base in base_by_owner.items():
        current = current_by_owner[owner]
        successor = successor_by_owner.get(owner)
        if successor is None or (successor != base and successor != current):
            return None
    return [copy.deepcopy(role) for role in successor_roles]


def serving_implementation_only_successor(
    active_bom: Mapping[str, Any], successor_bom: Mapping[str, Any], *,
    effective_active_roles: Sequence[Mapping[str, Any]] | None = None,
) -> bool:
    """Accept only a real serving-image refresh after Catalog activation."""
    if is_local_test3_bom(active_bom) or is_local_test3_bom(successor_bom):
        # Local image IDs are not an independently published Engine release.
        # Keep this non-release lane on the complete stopped Catalog boundary.
        return False
    # An unchanged release has nothing to publish. Runtime configuration uses
    # an owning L0 operation and must never restart the entire serving tuple.
    if canonical_bytes(dict(active_bom)) == canonical_bytes(dict(successor_bom)):
        return False
    replacements = {
        "runtime_api", "runtime_worker", "integration_service", "sfcrm_app_service",
    }
    if "mcp_worker" in active_bom.get("services", {}):
        replacements.add("mcp_worker")
    active_services = active_bom.get("services")
    successor_services = successor_bom.get("services")
    if not isinstance(active_services, dict) or not isinstance(successor_services, dict):
        return False
    if set(active_services) != set(successor_services) or not replacements.issubset(active_services):
        return False
    for role in replacements:
        active_service = active_services.get(role)
        successor_service = successor_services.get(role)
        if not isinstance(active_service, dict) or not isinstance(successor_service, dict):
            return False
        if (
            active_service.get("oci_repository") != successor_service.get("oci_repository")
            or active_service.get("source_repository") != successor_service.get("source_repository")
        ):
            return False
    service_changed = any(
        active_services[role] != successor_services[role] for role in replacements
    )
    base_roles = active_bom.get("runtime_worker_roles")
    successor_roles = successor_bom.get("runtime_worker_roles")
    accepted_roles = compatible_successor_worker_roles(
        base_roles,
        successor_roles,
        effective_active_roles,
    )
    if accepted_roles is None:
        return False
    # Runtime role tuning is mutable L0 serving configuration.  It must use
    # reconfigure-worker-role instead of manufacturing a successor BOM and
    # restarting the whole serving set.
    if not service_changed:
        return False
    successor_engine = successor_bom.get("l0_engine")
    active_engine = active_bom.get("l0_engine")
    if successor_engine is not None:
        if not isinstance(successor_engine, dict):
            return False
        successor_release = successor_engine.get("release")
        if not isinstance(successor_release, dict):
            return False
        successor_catalog = successor_release.get("catalog_reconcile")
        if active_engine is not None:
            if not isinstance(active_engine, dict) or not isinstance(active_engine.get("release"), dict):
                return False
            if active_engine["release"].get("catalog_reconcile") != successor_catalog:
                return False
        else:
            active_jobs = active_bom.get("jobs")
            successor_jobs = successor_bom.get("jobs")
            if not isinstance(active_jobs, dict) or not isinstance(successor_jobs, dict):
                return False
            if active_jobs.get("catalog_reconcile") != successor_jobs.get("catalog_reconcile"):
                return False
    projected = copy.deepcopy(dict(active_bom))
    projected["schema_version"] = successor_bom.get("schema_version")
    projected["desired_bom_sha256"] = successor_bom.get("desired_bom_sha256")
    projected["runtime_worker_roles"] = accepted_roles
    for role in replacements:
        projected["services"][role] = copy.deepcopy(successor_services[role])
    if successor_engine is not None:
        projected["l0_engine"] = copy.deepcopy(successor_engine)
    return canonical_bytes(projected) == canonical_bytes(dict(successor_bom))


def same_bom_governed_reference_data_completion(
    active_bom: Mapping[str, Any],
    successor_bom: Mapping[str, Any],
    *,
    phase: str,
    receipt: object,
) -> bool:
    """Allow one serving-state migration that supplies the newly required receipt."""
    return (
        phase == "serving"
        and not isinstance(receipt, dict)
        and canonical_bytes(dict(active_bom)) == canonical_bytes(dict(successor_bom))
    )


def same_bom_pre_publication_tool_refresh(
    active_bom: Mapping[str, Any],
    successor_bom: Mapping[str, Any],
    *,
    phase: str,
    prepared: Mapping[str, Any],
    replacement_manifest: Mapping[str, Any],
    replacement_manifest_digest: str,
) -> bool:
    """Accept one repaired schema tool after Worker activation, before publication.

    This is deliberately not a generic same-release retry.  The unchanged BOM
    proves that no serving, Catalog, AppRelease, frontend, or role input moved;
    the new v2 manifest proves that only the exact source-closed executable used
    to converge stopped L0 schema changed.
    """
    if phase != "serving-start-in-progress":
        return False
    if canonical_bytes(dict(active_bom)) != canonical_bytes(dict(successor_bom)):
        return False
    previous_digest = str(prepared.get("deploymentToolsManifestDigest", ""))
    if (
        not SHA256_RE.fullmatch(previous_digest)
        or not SHA256_RE.fullmatch(replacement_manifest_digest)
        or previous_digest == replacement_manifest_digest
    ):
        return False
    return replacement_manifest.get("schema") == DEPLOYMENT_TOOL_SCHEMA


def serving_catalog_successor(
    active_bom: Mapping[str, Any], successor_bom: Mapping[str, Any], *,
    effective_active_roles: Sequence[Mapping[str, Any]] | None = None,
    active_history: SealedHistoricalBOM | None = None,
) -> bool:
    """Accept one stopped release set plus its Catalog input, and nothing else.

    A private L0 capability added to an already-installed L1 app changes the
    immutable AppRelease grant as well as the binaries that consume/serve it.
    An L2 metadata correction changes the immutable downstream pack tree. Both
    use the existing stopped UPGRADE/active-release refresh path; neither
    requires a production-clone reset or a second Catalog mechanism.
    """
    if canonical_bytes(dict(active_bom)) == canonical_bytes(dict(successor_bom)):
        return False
    if is_local_test3_bom(active_bom) or is_local_test3_bom(successor_bom):
        # A sealed historical release may predate today's required capability
        # grants. Validate the new selection, not the old release against a
        # moving policy; its run-owned identity is checked by the caller.
        if validate_deployment_bom_view(dict(successor_bom)):
            return False
        if active_history is not None:
            if active_history.bom != dict(active_bom):
                return False
            active_desired = active_history.desired
        else:
            if is_local_test3_bom(active_bom) and validate_deployment_bom_view(dict(active_bom)):
                return False
            active_desired = desired_v10_projection_from_resolved(dict(active_bom))
        successor_desired = desired_v10_projection_from_resolved(dict(successor_bom))
        return (
            active_desired["execution"] == successor_desired["execution"]
            and active_desired["app_installation"]["app_id"]
            == successor_desired["app_installation"]["app_id"]
            and all(
                active_desired["app_installation"][field]["repository"]
                == successor_desired["app_installation"][field]["repository"]
                for field in ("artifact", "publisher_attestation", "publisher_verification_key")
            )
            and all(
                active_desired["l2_metadata"][field] == successor_desired["l2_metadata"][field]
                for field in ("repository", "path")
            )
            and compatible_successor_worker_roles(
                active_desired["runtime_worker_roles"],
                successor_desired["runtime_worker_roles"], effective_active_roles,
            ) is not None
        )
    active_services = active_bom.get("services")
    successor_services = successor_bom.get("services")
    if not isinstance(active_services, dict) or not isinstance(successor_services, dict):
        return False
    if set(active_services) != set(successor_services):
        return False
    for role in active_services:
        active_service = active_services.get(role)
        successor_service = successor_services.get(role)
        if not isinstance(active_service, dict) or not isinstance(successor_service, dict):
            return False
        if (
            active_service.get("oci_repository") != successor_service.get("oci_repository")
            or active_service.get("source_repository") != successor_service.get("source_repository")
        ):
            return False
    active_jobs = active_bom.get("jobs")
    successor_jobs = successor_bom.get("jobs")
    if not isinstance(active_jobs, dict) or not isinstance(successor_jobs, dict):
        return False
    if set(active_jobs) != {"catalog_reconcile"} or set(successor_jobs) != {"catalog_reconcile"}:
        return False
    active_catalog = active_jobs["catalog_reconcile"]
    successor_catalog = successor_jobs["catalog_reconcile"]
    if not isinstance(active_catalog, dict) or not isinstance(successor_catalog, dict):
        return False
    if (
        active_catalog.get("oci_repository") != successor_catalog.get("oci_repository")
        or active_catalog.get("source_repository") != successor_catalog.get("source_repository")
    ):
        return False
    active_app = active_bom.get("app_installation")
    successor_app = successor_bom.get("app_installation")
    if not isinstance(active_app, dict) or not isinstance(successor_app, dict):
        return False
    if active_app.get("app_id") != successor_app.get("app_id"):
        return False
    for owner_field in ("artifact", "publisher_attestation", "publisher_verification_key"):
        active_owner = active_app.get(owner_field)
        successor_owner = successor_app.get(owner_field)
        if not isinstance(active_owner, dict) or not isinstance(successor_owner, dict):
            return False
        if active_owner.get("repository") != successor_owner.get("repository"):
            return False
    active_l2 = active_bom.get("l2_metadata")
    successor_l2 = successor_bom.get("l2_metadata")
    if not isinstance(active_l2, dict) or not isinstance(successor_l2, dict):
        return False
    if (
        active_l2.get("repository") != successor_l2.get("repository")
        or active_l2.get("path") != successor_l2.get("path")
    ):
        return False
    active_frontend = active_bom.get("frontend")
    successor_frontend = successor_bom.get("frontend")
    if not isinstance(active_frontend, dict) or not isinstance(successor_frontend, dict):
        return False
    for owner_field in (
        "oci_repository", "nexuscrm_repository", "app_packs_repository",
        "l2_repository", "routing_mode",
    ):
        if active_frontend.get(owner_field) != successor_frontend.get(owner_field):
            return False
    base_roles = active_bom.get("runtime_worker_roles")
    successor_roles = successor_bom.get("runtime_worker_roles")
    accepted_roles = compatible_successor_worker_roles(
        base_roles,
        successor_roles,
        effective_active_roles,
    )
    if accepted_roles is None:
        return False
    projected = copy.deepcopy(dict(active_bom))
    projected["desired_bom_sha256"] = successor_bom.get("desired_bom_sha256")
    projected["runtime_worker_roles"] = accepted_roles
    if "l0_engine" in successor_bom:
        projected["l0_engine"] = copy.deepcopy(successor_bom["l0_engine"])
    else:
        projected.pop("l0_engine", None)
    for section in (
        "services", "jobs", "frontend", "app_installation",
        "catalog_consumers", "l2_metadata",
    ):
        value = successor_bom.get(section)
        if not isinstance(value, dict):
            return False
        projected[section] = copy.deepcopy(value)
    return canonical_bytes(projected) == canonical_bytes(dict(successor_bom))

def retained_frontend_target(state: Mapping[str, Any]) -> Path:
    """Return the frontend owner that an interrupted successor has not replaced."""
    current = dict(state)
    visited: set[Path] = set()
    for _ in range(16):
        for key in ("publishedFrontendTarget", "retainedFrontendTarget"):
            value = current.get(key)
            if isinstance(value, str) and value:
                return Path(value).resolve()
        transition = current.get("supersededTransition")
        if isinstance(transition, dict):
            archived_value = transition.get("archivedState")
            expected_run = transition.get("runId")
            if isinstance(archived_value, str) and archived_value and isinstance(expected_run, str):
                archived_path = Path(archived_value).resolve()
                if archived_path in visited:
                    raise DeployError("superseded D3 frontend owner contains an archive cycle")
                visited.add(archived_path)
                archived = load_json_object(archived_path, "superseded D3 state")
                if archived.get("schema") != STATE_SCHEMA or archived.get("runId") != expected_run:
                    raise DeployError("superseded D3 frontend owner has the wrong state identity")
                current = archived
                continue
        previous = current.get("previousFrontendTarget")
        if not isinstance(previous, str) or not previous:
            raise DeployError("D3 state has no retained frontend owner")
        return Path(previous).resolve()
    raise DeployError("superseded D3 frontend owner archive chain is too deep")


def supersede_serving_catalog_candidate(
    context: D3Context,
    *,
    active: Mapping[str, Any],
    active_bom: Mapping[str, Any],
    active_bom_path: Path,
    expected_active_run_id: str,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> dict[str, Any]:
    """Start a stopped incremental Catalog refresh on the retained database."""
    history = read_sealed_historical_bom(active)
    if history.bom != dict(active_bom) or history.path != active_bom_path:
        raise DeployError("serving Catalog successor lost its sealed predecessor identity")
    # Validate every candidate-owned artifact and both compatibility boundaries
    # before stopping the serving tuple. A bad candidate must not create an
    # outage merely so preparation can discover that it never could run.
    preflight_candidate_artifacts(
        context,
        app_packs_root=app_packs_root,
        core_root=core_root,
        l2_root=l2_root,
        execute=execute,
        capture_output=capture_output,
    )
    previous_ownership = require_worker_ownership_after(active)
    if read_worker_ownership(context.envv) != previous_ownership:
        raise DeployError("serving Catalog successor refuses changed Worker authority")
    previous_frontend = Path(str(active.get("previousFrontendTarget", ""))).resolve()
    expected_live_frontend = retained_frontend_target(active)
    if resolve_frontend_dist_target(
        runtime_paths(context.env_root)["frontend_dist"]
    ) != expected_live_frontend:
        raise DeployError("serving Catalog successor refuses a changed frontend owner")

    active_context = D3Context(
        env_file=context.env_file,
        envv=context.envv,
        env_root=context.env_root,
        resolved_bom=active_bom_path,
        bom=dict(active_bom),
        bom_file_digest=sha256_bytes(active_bom_path.read_bytes()),
        run_id=expected_active_run_id,
        run_root=Path(str(active.get("runRoot", ""))),
        state_file=context.state_file,
        ports=derive_ports(context.envv, worker_roles_from_bom(active_bom)[0]),
        images=history.images,
        worker_stems=worker_roles_from_bom(active_bom)[0],
        worker_families=worker_roles_from_bom(active_bom)[1],
    )
    # The live revision still belongs to the active candidate.  The successor
    # AppRelease has not been activated yet and must not be used as the
    # pre-mutation authority check.
    verify_catalog_after(active_context, active)
    expected_images = container_images_by_name(active_context, active)
    present = set(owned_containers())
    if not present.issubset(expected_images):
        raise DeployError("serving Catalog successor found an unknown D3 container")
    for name in reversed(expected_serving_container_order(active)):
        if name not in present:
            continue
        if verify_container_identity(
            active_context, name, expected_images[name], capture_output=capture_output,
        ):
            execute(("docker", "stop", "--time", "30", name))
        execute(("docker", "rm", name))

    if context.run_root.is_symlink() or (
        context.run_root.exists()
        and (not context.run_root.is_dir() or any(context.run_root.iterdir()))
    ):
        raise DeployError(f"D3 AppRelease successor run root is not empty: {context.run_root}")
    state_root = context.state_file.parent
    successor_suffix = hashlib.sha256(context.run_id.encode()).hexdigest()[:12]
    archived = state_root / f"superseded-{expected_active_run_id}-{successor_suffix}.json"
    archived_payload = {
        **dict(active),
        "supersededBy": {
            "runId": context.run_id,
            "mode": "serving-catalog-upgrade",
            "resolvedBomFileDigest": context.bom_file_digest,
            "desiredBomDigest": context.bom["desired_bom_sha256"],
        },
    }
    archived_bytes = canonical_bytes(archived_payload)
    if archived.is_symlink() or (archived.exists() and not archived.is_file()):
        raise DeployError(f"D3 AppRelease successor evidence is not a regular file: {archived}")
    if archived.exists() and archived.read_bytes() != archived_bytes:
        raise DeployError(f"D3 AppRelease successor evidence conflicts: {archived}")
    if not archived.exists():
        pending_archive = archived.with_suffix(".json.pending")
        if pending_archive.exists() or pending_archive.is_symlink():
            raise DeployError(f"stale D3 AppRelease successor evidence exists: {pending_archive}")
        pending_archive.write_bytes(archived_bytes)
        os.chmod(pending_archive, 0o600)
        os.replace(pending_archive, archived)

    context.run_root.mkdir(parents=True, mode=0o700, exist_ok=True)
    unit_states = active.get("previousSystemdUnits")
    if not isinstance(unit_states, dict):
        raise DeployError("serving Catalog successor lost systemd rollback authority")
    next_state = state_payload(
        context,
        previous_frontend_target=previous_frontend,
        previous_systemd_units=unit_states,
    )
    for key in ("previousNginxConfig", "previousNginxConfigDigest"):
        if key in active:
            next_state[key] = active[key]
    next_state["retainedFrontendTarget"] = str(expected_live_frontend)
    next_state["supersededTransition"] = {
        "runId": expected_active_run_id,
        "phase": active.get("phase"),
        "mode": "serving-catalog-upgrade",
        "archivedState": str(archived),
    }
    pending = context.state_file.with_suffix(".json.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError(f"stale D3 AppRelease successor state exists: {pending}")
    pending.write_bytes(canonical_bytes(next_state))
    os.chmod(pending, 0o600)
    os.replace(pending, context.state_file)
    return next_state


def same_bom_service_image_cleanup(
    active_bom: Mapping[str, Any],
    successor_bom: Mapping[str, Any],
    *,
    phase: str,
    state: Mapping[str, Any],
) -> bool:
    """Allow an exact signed BOM to retire only transient service-image state."""
    if canonical_bytes(dict(active_bom)) != canonical_bytes(dict(successor_bom)):
        return False
    overrides = state.get("serviceImageOverrides", {})
    operations = state.get("serviceImageOperations", [])
    transition = state.get("serviceImageTransition")
    if not isinstance(overrides, dict) or not isinstance(operations, list):
        return False
    if phase == "serving":
        return transition is None and bool(overrides)
    if phase == "service-image-replace-in-progress":
        return isinstance(transition, dict)
    return False


def service_image_transition_candidate_images(
    context: D3Context,
    state: Mapping[str, Any],
) -> dict[str, str] | None:
    """Validate the alternate container identities owned by an interrupted cutover."""
    if state.get("phase") != "service-image-replace-in-progress":
        return None
    transition = state.get("serviceImageTransition")
    roles = ("runtime_api", "integration_service", "sfcrm_app_service")
    role_set = set(roles)
    if (
        not isinstance(transition, dict)
        or transition.get("schema") != L0_SERVICE_IMAGE_TRANSITION_SCHEMA
    ):
        raise DeployError("service image cleanup found an invalid active transition")
    previous = transition.get("previousImages")
    candidate = transition.get("candidateImages")
    requested = transition.get("requestedRoles")
    if (
        not isinstance(previous, dict)
        or set(previous) != role_set
        or not isinstance(candidate, dict)
        or set(candidate) != role_set
        or not isinstance(requested, list)
        or not requested
        or any(not isinstance(role, str) for role in requested)
        or requested != sorted(set(requested))
        or not set(requested).issubset(role_set)
        or previous != effective_service_images(context, state)
    ):
        raise DeployError("service image cleanup found an inconsistent active transition")
    for role in roles:
        repository = context.images[role].partition("@sha256:")[0]
        for image in (previous[role], candidate[role]):
            if not isinstance(image, str) or not re.fullmatch(
                re.escape(repository) + r"@sha256:[0-9a-f]{64}", image,
            ):
                raise DeployError("service image cleanup found an unowned image")
        if role not in requested and candidate[role] != previous[role]:
            raise DeployError("service image cleanup changed an unrequested role")
    return dict(candidate)


def supersede_serving_implementation_candidate(
    context: D3Context,
    *,
    active: Mapping[str, Any],
    expected_active_run_id: str,
    core_root: Path | None = None,
    app_packs_root: Path | None = None,
    l2_root: Path | None = None,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> dict[str, Any]:
    """Refresh L0/serving images without replaying an unchanged Catalog."""
    prepared = active.get("prepared")
    if not isinstance(prepared, dict):
        raise DeployError("serving implementation successor has no sealed prepared artifacts")
    # The sealed file remains the one-owner v10 release authority.  Successor
    # comparison needs the same read-only member projection used for the new
    # candidate; never make a subsequent v10 rollout depend on the removed
    # four-publication grammar.
    history = read_sealed_historical_bom(active)
    active_bom_path, active_bom = history.path, history.bom
    phase = str(active.get("phase", ""))
    effective_active_roles = None
    if phase in {"serving", "service-image-replace-in-progress"} and active.get(
        "workerRoleConfigOverrides"
    ):
        effective_active_roles = effective_runtime_worker_role_policies(
            active, active_bom,
        )
    ownership_before_phase = phase in {
        "catalog-applied", "governed-reference-data-in-progress",
        "governed-reference-data-applied", "serving-inputs-ready",
        "worker-activation-in-progress",
    }
    deployment_tools = context.resolved_bom.parent / "deployment-tools"
    replacement_tool_manifest: dict[str, Any] | None = None
    replacement_tool_manifest_digest = ""
    same_bom_tool_refresh = False
    same_bom_reference_data_completion = (
        same_bom_governed_reference_data_completion(
            active_bom,
            context.bom,
            phase=phase,
            receipt=active.get("governedReferenceData"),
        )
    )
    same_bom = canonical_bytes(dict(active_bom)) == canonical_bytes(dict(context.bom))
    same_bom_service_cleanup = same_bom_service_image_cleanup(
        active_bom, context.bom, phase=phase, state=active,
    )
    if same_bom:
        try:
            replacement_tool_manifest = verify_deployment_tools(
                root=deployment_tools, resolved_bom=context.resolved_bom,
            )
        except DeploymentToolError as exc:
            raise DeployError(
                f"same-BOM pre-publication deployment-tools bundle is invalid: {exc}"
            ) from exc
        replacement_tool_manifest_digest = sha256_bytes(
            (deployment_tools / "manifest.json").read_bytes()
        )
        same_bom_tool_refresh = same_bom_pre_publication_tool_refresh(
            active_bom,
            context.bom,
            phase=phase,
            prepared=prepared,
            replacement_manifest=replacement_tool_manifest,
            replacement_manifest_digest=replacement_tool_manifest_digest,
        )
    if not (
        serving_implementation_only_successor(
            active_bom,
            context.bom,
            effective_active_roles=effective_active_roles,
        )
        or same_bom_tool_refresh
        or same_bom_reference_data_completion
        or same_bom_service_cleanup
    ):
        if serving_catalog_successor(
            active_bom,
            context.bom,
            effective_active_roles=effective_active_roles,
            active_history=history,
        ):
            if core_root is None or app_packs_root is None or l2_root is None:
                raise DeployError(
                    "serving Catalog successor requires exact Core, AppPackage, and L2 sources"
                )
            return supersede_serving_catalog_candidate(
                context,
                active=active,
                active_bom=active_bom,
                active_bom_path=active_bom_path,
                expected_active_run_id=expected_active_run_id,
                app_packs_root=app_packs_root,
                core_root=core_root,
                l2_root=l2_root,
                execute=execute,
                capture_output=capture_output,
            )
        raise DeployError(
            "serving successor permits only exact serving images, transient "
            "same-BOM cleanup, or one stopped Catalog upgrade"
        )
    if same_bom_tool_refresh:
        successor_mode = "same-bom-pre-publication-tool-refresh"
    elif same_bom_reference_data_completion:
        successor_mode = "same-bom-governed-reference-data-completion"
    elif same_bom_service_cleanup:
        successor_mode = "same-bom-service-image-cleanup"
    elif ownership_before_phase:
        successor_mode = "pre-serving-implementation-refresh"
    else:
        successor_mode = "serving-implementation-refresh"
    # Every serving-image successor follows the same complete, read-only proof
    # as a Catalog successor before any current container is stopped. This
    # includes Engine/App compatibility and the exact frontend distribution.
    if not same_bom_tool_refresh and not same_bom_reference_data_completion:
        if core_root is None or app_packs_root is None or l2_root is None:
            raise DeployError(
                "serving implementation successor requires exact Core, AppPackage, and L2 sources"
            )
        preflight_candidate_artifacts(
            context,
            app_packs_root=app_packs_root,
            core_root=core_root,
            l2_root=l2_root,
            execute=execute,
            capture_output=capture_output,
        )
    active_context = D3Context(
        env_file=context.env_file,
        envv=context.envv,
        env_root=context.env_root,
        resolved_bom=active_bom_path,
        bom=active_bom,
        bom_file_digest=sha256_bytes(active_bom_path.read_bytes()),
        run_id=expected_active_run_id,
        run_root=Path(str(active.get("runRoot", ""))),
        state_file=context.state_file,
        ports=derive_ports(context.envv, worker_roles_from_bom(active_bom)[0]),
        images=history.images,
        worker_stems=worker_roles_from_bom(active_bom)[0],
        worker_families=worker_roles_from_bom(active_bom)[1],
    )
    transition_candidates = service_image_transition_candidate_images(
        active_context, active,
    )
    verify_catalog_after(active_context, active)
    if ownership_before_phase:
        worker_table_present, previous_ownership = read_optional_worker_ownership(
            context.envv,
        )
        if not worker_table_present or previous_ownership != require_worker_ownership_before(active):
            raise DeployError(
                "pre-serving implementation successor refuses changed Worker authority"
            )
    else:
        previous_ownership = require_worker_ownership_after(active)
        if read_worker_ownership(context.envv) != previous_ownership:
            raise DeployError("serving implementation successor refuses changed Worker authority")
    expected_live_frontend = retained_frontend_target(active)
    if resolve_frontend_dist_target(
        runtime_paths(context.env_root)["frontend_dist"]
    ) != expected_live_frontend:
        raise DeployError("serving implementation successor refuses a changed frontend owner")
    if context.run_root.is_symlink() or (
        context.run_root.exists()
        and (not context.run_root.is_dir() or any(context.run_root.iterdir()))
    ):
        raise DeployError(f"D3 serving successor run root is not empty: {context.run_root}")

    schema_refresh = ownership_before_phase or same_bom_tool_refresh
    if schema_refresh:
        if core_root is None or app_packs_root is None:
            raise DeployError(
                "pre-serving implementation successor requires exact Core and AppRelease sources"
            )
        core_commit = context.bom["services"]["runtime_api"]["source_commit"]
        core_root = require_exact_checkout(core_root, core_commit, "Core")
        app_packs_root = require_exact_checkout(
            app_packs_root,
            context.bom["app_installation"]["artifact"]["commit"],
            "AppRelease",
        )
        if replacement_tool_manifest is None:
            try:
                replacement_tool_manifest = verify_deployment_tools(
                    root=deployment_tools, resolved_bom=context.resolved_bom,
                )
            except DeploymentToolError as exc:
                raise DeployError(
                    f"pre-serving successor deployment-tools bundle is invalid: {exc}"
                ) from exc
            replacement_tool_manifest_digest = sha256_bytes(
                (deployment_tools / "manifest.json").read_bytes()
            )

    expected_images = container_images_by_name(active_context, active)
    candidate_images_by_name: dict[str, str] = {}
    if transition_candidates is not None:
        containers = active.get("containers")
        if not isinstance(containers, dict):
            raise DeployError("service image cleanup lost serving container identities")
        for role, container_key in (
            ("runtime_api", "runtime_api"),
            ("integration_service", "integration_service"),
            ("sfcrm_app_service", "sfcrm_app_service"),
        ):
            name = containers.get(container_key)
            if not isinstance(name, str) or not name:
                raise DeployError("service image cleanup lost one serving container identity")
            candidate_images_by_name[name] = transition_candidates[role]
    present = set(owned_containers())
    if not present.issubset(expected_images):
        raise DeployError("serving implementation successor found an unknown D3 container")
    for name in reversed(expected_serving_container_order(active)):
        if name not in present:
            continue
        candidate_image = candidate_images_by_name.get(name)
        if candidate_image is None:
            running = verify_container_identity(
                active_context, name, expected_images[name], capture_output=capture_output,
            )
        else:
            fields = capture_output((
                "docker", "inspect", "--format",
                '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}\t'
                '{{.HostConfig.NetworkMode}}\t{{.HostConfig.RestartPolicy.Name}}\t{{.State.Running}}',
                name,
            )).split("\t")
            if (
                len(fields) != 5
                or fields[0] not in {expected_images[name], candidate_image}
                or fields[1:4] != [active_context.run_id, "host", "unless-stopped"]
                or fields[4] not in {"true", "false"}
            ):
                raise DeployError(
                    f"service image cleanup found a drifted container: {name}"
                )
            running = fields[4] == "true"
        if running:
            execute(("docker", "stop", "--time", "30", name))
        execute(("docker", "rm", name))

    schema_receipt: dict[str, Any] | None = None
    if schema_refresh:
        setup_binary = deployment_tools / DEPLOYMENT_TOOL_FILES["setup"]
        runtime_env = dict(context.envv)
        runtime_env["NEXUSCRM_SETUP_BIN"] = str(setup_binary)
        runtime_env["NEXUSCRM_OFFICIAL_APP_PACKS_ROOT"] = str(app_packs_root)
        run_backend_setup(
            runtime_env, core_root, app_packs_root, ["ensure-l0-schema-ddl"],
        )
        schema_receipt = {
            "command": "ensure-l0-schema-ddl",
            "coreCommit": core_commit,
            "setupCoreRevision": replacement_tool_manifest["coreRevision"],
            "setupControlplaneRevision": replacement_tool_manifest.get(
                "controlplaneRevision"
            ),
            "deploymentToolsManifestDigest": replacement_tool_manifest_digest,
        }

    state_root = context.state_file.parent
    successor_suffix = hashlib.sha256(context.run_id.encode()).hexdigest()[:12]
    archived = state_root / f"superseded-{expected_active_run_id}-{successor_suffix}.json"
    archived_payload = {
        **dict(active),
        "supersededBy": {
            "runId": context.run_id,
            "mode": successor_mode,
            "resolvedBomFileDigest": context.bom_file_digest,
            "desiredBomDigest": context.bom["desired_bom_sha256"],
        },
    }
    archived_bytes = canonical_bytes(archived_payload)
    if archived.is_symlink() or (archived.exists() and not archived.is_file()):
        raise DeployError(f"D3 serving successor evidence is not a regular file: {archived}")
    if archived.exists() and archived.read_bytes() != archived_bytes:
        raise DeployError(f"D3 serving successor evidence conflicts: {archived}")
    if not archived.exists():
        pending_archive = archived.with_suffix(".json.pending")
        if pending_archive.exists() or pending_archive.is_symlink():
            raise DeployError(f"stale D3 serving successor evidence exists: {pending_archive}")
        pending_archive.write_bytes(archived_bytes)
        os.chmod(pending_archive, 0o600)
        os.replace(pending_archive, archived)

    context.run_root.mkdir(parents=True, mode=0o700, exist_ok=True)
    copied_bom = context.run_root / "resolved-bom.json"
    copied_bom.write_bytes(context.resolved_bom.read_bytes())
    os.chmod(copied_bom, 0o400)
    successor_prepared = {**prepared, "resolvedBomCopy": str(copied_bom)}
    if schema_refresh:
        successor_tool_root = context.run_root / "authority" / "deployment-tools"
        shutil.copytree(deployment_tools, successor_tool_root, symlinks=False)
        try:
            verified_successor_tools = verify_deployment_tools(
                root=successor_tool_root, resolved_bom=copied_bom,
            )
        except DeploymentToolError as exc:
            raise DeployError(
                f"copied successor deployment-tools bundle is invalid: {exc}"
            ) from exc
        successor_setup = successor_tool_root / DEPLOYMENT_TOOL_FILES["setup"]
        successor_metadata = (
            successor_tool_root / DEPLOYMENT_TOOL_FILES["pack_runtime_metadata"]
        )
        os.chmod(successor_setup, 0o500)
        os.chmod(successor_metadata, 0o500)
        os.chmod(successor_tool_root / "manifest.json", 0o400)
        successor_prepared.update({
            "setupBinary": str(successor_setup),
            "packRuntimeMetadataBinary": str(successor_metadata),
            "deploymentToolsManifestDigest": replacement_tool_manifest_digest,
            "setupCoreRevision": str(verified_successor_tools["coreRevision"]),
        })
    successor_containers = {
        "runtime_api": container_name(context.run_id, "runtime-api"),
        "integration_service": container_name(context.run_id, "integration"),
        "sfcrm_app_service": container_name(context.run_id, "sfcrm"),
        **({"mcp_worker": container_name(context.run_id, "mcp")} if "mcp_worker" in context.images else {}),
        "workers": {
            stem: container_name(context.run_id, f"worker-{stem}")
            for stem in context_worker_stems(context)
        },
    }
    stale_keys = {
        "servingArtifacts", "workerRoleConfigDigests", "workerOwnershipAfter",
        "startedContainers", "publishedFrontendTarget", "rollbackStoppedContainers",
        "publishedFrontendDigest", "publishedNginxConfigDigest", "deploymentEvidence",
        "rollbackFrontendTarget", "workerRoleConfigOverrides",
        "runtimeWorkerConfigOperations", "runtimeWorkerConfigTransition",
        "workerImageOverride", "workerImageSourceRevisionOverride",
        "workerImageOperations", "workerImageTransition",
        "serviceImageOverrides", "serviceImageOperations",
        "serviceImageTransition",
        "governedReferenceData",
        "releaseClass", "controllerCommit",
    }
    next_state = {key: value for key, value in active.items() if key not in stale_keys}
    next_state.update({
        "phase": "catalog-applied",
        "runId": context.run_id,
        "runRoot": str(context.run_root),
        "resolvedBomPath": str(copied_bom),
        "resolvedBomSchema": context.bom["schema_version"],
        "resolvedBomFileDigest": context.bom_file_digest,
        "desiredBomDigest": context.bom["desired_bom_sha256"],
        **local_deployment_evidence(context.bom),
        "images": dict(context.images),
        "prepared": successor_prepared,
        "containers": successor_containers,
        "workerOwnershipTableBefore": True,
        "workerOwnershipBefore": worker_ownership_payload(previous_ownership),
        "retainedFrontendTarget": str(expected_live_frontend),
        "supersededTransition": {
            "runId": expected_active_run_id,
            "phase": active.get("phase"),
            "mode": successor_mode,
            "archivedState": str(archived),
        },
    })
    if schema_receipt is not None:
        next_state["runtimeL0SchemaConvergence"] = schema_receipt
        exact_sources = next_state.get("exactSources")
        if isinstance(exact_sources, dict):
            next_state["exactSources"] = {
                **exact_sources,
                "coreRoot": str(core_root),
                "appPacksRoot": str(app_packs_root),
                "coreCommit": schema_receipt["coreCommit"],
                "appPacksCommit": context.bom["app_installation"]["artifact"]["commit"],
            }
    pending = context.state_file.with_suffix(".json.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError(f"stale D3 serving successor state exists: {pending}")
    pending.write_bytes(canonical_bytes(next_state))
    os.chmod(pending, 0o600)
    os.replace(pending, context.state_file)
    return next_state


def catalog_retry_operation_id(
    envv: Mapping[str, str], config: Mapping[str, Any],
    *, query: Callable[..., str] = db_query_scalar,
) -> str:
    """Prove a failed Catalog attempt has no immutable serving selection yet."""
    required = {
        "orgId": config.get("orgId"),
        "appId": config.get("appId"),
        "idempotencyKey": config.get("idempotencyKey"),
    }
    for label, value in required.items():
        if not isinstance(value, str) or not value or len(value.encode("utf-8")) > 255:
            raise DeployError(f"sealed Catalog config has invalid {label}")

    def literal(value: str) -> str:
        return f"CONVERT(X'{value.encode('utf-8').hex()}' USING utf8mb4)"

    sql = f"""
SELECT CONCAT_WS('\t', HEX(o.OperationId), o.State, o.Step,
  CAST(COALESCE(SUM(j.Step = 'METADATA_SELECTION_COMMITTED'), 0) AS CHAR),
  CAST(COALESCE(SUM(j.Step IN ('SERVING_ARTIFACTS_COMMITTED',
    'PROJECTION_ACKNOWLEDGED', 'ACTIVATED')), 0) AS CHAR))
FROM _System_CatalogAppOperation o
LEFT JOIN _System_CatalogAppOperationJournal j ON j.OperationId = o.OperationId
WHERE o.OrgId = {literal(required['orgId'])}
  AND o.AppId = {literal(required['appId'])}
  AND o.IdempotencyKey = {literal(required['idempotencyKey'])}
GROUP BY o.OperationId, o.State, o.Step;
""".strip()
    raw = query(dict(envv), sql, database=envv["TIDB_DATABASE"])
    rows = raw.splitlines() if raw else []
    # Config/capability validation can fail before BeginOperation.  In that
    # case there is deliberately no operation to preserve and the same sealed
    # config may be retried with the corrected implementation.  Once an
    # operation exists, retain the stricter no-publication proof below.
    if not rows:
        return ""
    if len(rows) != 1:
        raise DeployError("Catalog implementation retry found multiple sealed operations")
    fields = rows[0].split("\t")
    if len(fields) != 5:
        raise DeployError("Catalog implementation retry operation has an unknown shape")
    operation_id = decode_hex_text(fields[0], "Catalog retry OperationId")
    expected_state = {
        "PREPARE": "PREPARING",
        "UPGRADE": "UPGRADING",
    }.get(config.get("operationKind"))
    if (
        expected_state is None
        or fields[1] != expected_state
        or fields[2] != "CANDIDATE_COMMITTED"
        or fields[3:] != ["0", "0"]
    ):
        raise DeployError(
            "Catalog implementation retry refuses an operation with committed publication state"
        )
    return operation_id


def completed_catalog_abandonment_evidence(value: object) -> str:
    """Validate and return the immutable root evidence of a completed abandonment."""
    required = {
        "schema", "operationId", "previousDesiredBomDigest",
        "replacementDesiredBomDigest", "reason", "status", "evidenceDigest",
        "output", "outputDigest",
    }
    if not isinstance(value, dict) or set(value) != required:
        raise DeployError("completed Catalog abandonment receipt is malformed")
    intent = {
        key: value[key]
        for key in (
            "schema", "operationId", "previousDesiredBomDigest",
            "replacementDesiredBomDigest", "reason",
        )
    }
    evidence_digest = value.get("evidenceDigest")
    if (
        value.get("schema") != "nexuscrm.test2-catalog-abandonment-intent/v1"
        or value.get("reason")
        != "replace-failed-unpublished-upgrade-with-signed-successor"
        or value.get("status") != "completed"
        or not isinstance(value.get("operationId"), str)
        or not re.fullmatch(
            r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
            value["operationId"],
        )
        or not isinstance(value.get("previousDesiredBomDigest"), str)
        or not SHA256_RE.fullmatch(value["previousDesiredBomDigest"])
        or not isinstance(value.get("replacementDesiredBomDigest"), str)
        or not SHA256_RE.fullmatch(value["replacementDesiredBomDigest"])
        or not isinstance(evidence_digest, str)
        or not SHA256_RE.fullmatch(evidence_digest)
        or sha256_bytes(canonical_bytes(intent)) != evidence_digest
        or not isinstance(value.get("outputDigest"), str)
        or not SHA256_RE.fullmatch(value["outputDigest"])
    ):
        raise DeployError("completed Catalog abandonment receipt is inconsistent")
    output_path = Path(str(value.get("output", "")))
    if (
        output_path.is_symlink() or not output_path.is_file()
        or sha256_bytes(output_path.read_bytes()) != value["outputDigest"]
    ):
        raise DeployError("completed Catalog abandonment lost its exact output")
    return evidence_digest


def supersede_unactivated_catalog_candidate(
    context: D3Context,
    *,
    active: Mapping[str, Any],
    expected_active_run_id: str,
) -> dict[str, Any]:
    """Retry unchanged L1/L2 content with latest L0 implementations.

    A Catalog-job-only correction may retain an already-created unactivated
    operation and its sealed config. Any other L0 implementation refresh is
    accepted only before BeginOperation. A failed stopped Runtime apply may
    also be replaced before Catalog preparation; its old L0 receipts are
    discarded and the apply resumes with rebuilt run-owned artifacts while the
    already-reset database is retained.
    """
    if owned_containers():
        raise DeployError("Catalog implementation retry refuses active D3 containers")
    prepared = active.get("prepared")
    artifacts = active.get("catalogArtifacts")
    active_phase = str(active.get("phase", ""))
    runtime_apply_refresh = (
        active_phase in {
            "quiesced", "runtime-apply-in-progress", "runtime-candidate-applied",
            "runtime-applied",
        }
        and artifacts is None
    )
    if not isinstance(prepared, dict):
        raise DeployError("L0 implementation retry has no prepared artifacts")
    if not runtime_apply_refresh and not isinstance(artifacts, dict):
        raise DeployError("Catalog implementation retry has no sealed artifacts")
    history = read_sealed_historical_bom(active)
    active_bom_path, active_bom = history.path, history.bom
    operation_id = ""
    sealed_config: dict[str, Any] = {}
    if not runtime_apply_refresh:
        assert isinstance(artifacts, dict)
        config_path = Path(str(artifacts.get("config", "")))
        config_digest = str(artifacts.get("configDigest", ""))
        if (
            config_path.is_symlink() or not config_path.is_file()
            or sha256_bytes(config_path.read_bytes()) != config_digest
        ):
            raise DeployError("Catalog implementation retry lost its sealed config")
        sealed_config = load_json_object(config_path, "sealed Catalog retry config")
        operation_id = catalog_retry_operation_id(context.envv, sealed_config)
    # Before BeginOperation there is no Catalog state to preserve.  The files
    # under runRoot are generated inputs, not a durable candidate, so replace
    # them from the current signed release instead of classifying the change as
    # an exact retry, implementation refresh, or release replacement.
    preoperation_replacement = not operation_id
    catalog_only = (
        bool(operation_id)
        and catalog_implementation_only_successor(active_bom, context.bom)
    )
    release_replacement = (
        preoperation_replacement
        and serving_catalog_successor(active_bom, context.bom, active_history=history)
    )
    abandonment_replacement = (
        bool(operation_id)
        and sealed_config.get("operationKind") == "UPGRADE"
        and serving_catalog_successor(active_bom, context.bom, active_history=history)
    )
    if not (preoperation_replacement or catalog_only or abandonment_replacement):
        raise DeployError(
            "Catalog successor refuses to replace a durable operation unless it is an "
            "exact Catalog implementation retry or an unpublished upgrade abandonment"
        )
    successor_mode = (
        "abandon-unpublished-catalog-upgrade"
        if abandonment_replacement else
        "unactivated-catalog-implementation-retry"
        if catalog_only else
        "preoperation-current-release-replacement"
    )
    prior_abandonment = active.get("catalogAbandonment")
    prior_abandonment_evidence = ""
    if prior_abandonment is not None and not abandonment_replacement:
        if (
            not isinstance(prior_abandonment, dict)
            or prior_abandonment.get("status") != "completed"
        ):
            raise DeployError(
                "resume the pending Catalog abandonment before replacing its signed successor"
            )
        prior_abandonment_evidence = completed_catalog_abandonment_evidence(
            prior_abandonment,
        )
    existing_abandonment_evidence = active.get(
        "catalogAbandonmentEvidenceDigest", "",
    )
    if (
        not isinstance(existing_abandonment_evidence, str)
        or (
            existing_abandonment_evidence
            and not SHA256_RE.fullmatch(existing_abandonment_evidence)
        )
        or (
            prior_abandonment_evidence
            and existing_abandonment_evidence
            and prior_abandonment_evidence != existing_abandonment_evidence
        )
    ):
        raise DeployError("Catalog abandonment history is inconsistent")
    abandonment_evidence = (
        prior_abandonment_evidence or existing_abandonment_evidence
    )

    if context.run_root.is_symlink() or (
        context.run_root.exists()
        and (not context.run_root.is_dir() or any(context.run_root.iterdir()))
    ):
        raise DeployError(f"D3 Catalog successor run root is not empty: {context.run_root}")
    state_root = context.state_file.parent
    successor_suffix = hashlib.sha256(context.run_id.encode()).hexdigest()[:12]
    archived = state_root / f"superseded-{expected_active_run_id}-{successor_suffix}.json"
    archived_payload = {
        **dict(active),
        "supersededBy": {
            "runId": context.run_id,
            "mode": successor_mode,
            "resolvedBomFileDigest": context.bom_file_digest,
            "desiredBomDigest": context.bom["desired_bom_sha256"],
        },
    }
    archived_bytes = canonical_bytes(archived_payload)
    if archived.is_symlink() or (archived.exists() and not archived.is_file()):
        raise DeployError(f"D3 Catalog successor evidence is not a regular file: {archived}")
    if archived.exists() and archived.read_bytes() != archived_bytes:
        raise DeployError(f"D3 Catalog successor evidence conflicts: {archived}")
    if not archived.exists():
        pending_archive = archived.with_suffix(".json.pending")
        if pending_archive.exists() or pending_archive.is_symlink():
            raise DeployError(f"stale D3 Catalog successor evidence exists: {pending_archive}")
        pending_archive.write_bytes(archived_bytes)
        os.chmod(pending_archive, 0o600)
        os.replace(pending_archive, archived)

    context.run_root.mkdir(parents=True, mode=0o700, exist_ok=True)
    copied_bom = context.run_root / "resolved-bom.json"
    successor_prepared: dict[str, Any] | None = None
    if catalog_only:
        copied_bom.write_bytes(context.resolved_bom.read_bytes())
        os.chmod(copied_bom, 0o400)
        successor_prepared = {**prepared, "resolvedBomCopy": str(copied_bom)}
    successor_containers = {
        "runtime_api": container_name(context.run_id, "runtime-api"),
        "integration_service": container_name(context.run_id, "integration"),
        "sfcrm_app_service": container_name(context.run_id, "sfcrm"),
        **({"mcp_worker": container_name(context.run_id, "mcp")} if "mcp_worker" in context.images else {}),
        "workers": {
            stem: container_name(context.run_id, f"worker-{stem}")
            for stem in context_worker_stems(context)
        },
    }
    next_state = {
        **{
            key: value for key, value in active.items()
            if key not in {"releaseClass", "controllerCommit"}
        },
        "phase": (
            "runtime-successor-captured" if runtime_apply_refresh else
            "catalog-in-progress" if catalog_only else
            "catalog-successor-captured"
        ),
        "runId": context.run_id,
        "runRoot": str(context.run_root),
        "resolvedBomPath": str(copied_bom if catalog_only else context.resolved_bom),
        "resolvedBomSchema": context.bom["schema_version"],
        "resolvedBomFileDigest": context.bom_file_digest,
        "desiredBomDigest": context.bom["desired_bom_sha256"],
        **local_deployment_evidence(context.bom),
        "images": dict(context.images),
        "ports": context.ports.all_ports(),
        "workerStems": list(context.worker_stems),
        "containers": successor_containers,
        "supersededTransition": {
            "runId": expected_active_run_id,
            "phase": active_phase,
            "mode": successor_mode,
            "archivedState": str(archived),
        },
    }
    if release_replacement or abandonment_replacement:
        start = require_catalog_start(active)
        if start["mode"] == "UPGRADE":
            before = require_catalog_before(active)
            replacement_release = context.bom["app_installation"][
                "app_release_digest"
            ]
            next_state["catalogStart"] = {
                "mode": "UPGRADE",
                "refreshActiveRelease": (
                    before.active_release_digest == replacement_release
                ),
            }
    if catalog_only:
        next_state["prepared"] = successor_prepared
        next_state["catalogImplementationRetry"] = {
            "operationId": operation_id,
            "previousImage": active_bom["jobs"]["catalog_reconcile"]["digest"],
            "replacementImage": context.bom["jobs"]["catalog_reconcile"]["digest"],
        }
    else:
        stale_fields = [
            "prepared",
            "exactSources",
            "catalogArtifacts",
            # Both receipts are bound to the superseded Catalog attempt.  The
            # archived state retains them for audit, while the successor must
            # derive fresh receipts from its own signed BOM and sealed config.
            "catalogImplementationRetry",
            "catalogPublisherTrustTransition",
            "catalogAbandonment",
        ]
        if runtime_apply_refresh:
            stale_fields.extend((
                "runtimeApply", "runtimeCandidateApply", "runtimeL0SchemaConvergence",
                "runtimeAppShell",
            ))
        for stale in stale_fields:
            next_state.pop(stale, None)
        if abandonment_evidence:
            next_state["catalogAbandonmentEvidenceDigest"] = abandonment_evidence
        next_state["catalogPreoperationSuccessor"] = {
            "catalogOperationId": "",
            "mode": successor_mode,
            "catalogVersions": {
                "l1": context.bom["app_installation"]["semantic_version"],
                "l2Tree": context.bom["l2_metadata"]["tree_sha256"],
            },
            "previousDesiredBomDigest": active_bom["desired_bom_sha256"],
            "replacementDesiredBomDigest": context.bom["desired_bom_sha256"],
            "resumeRuntimeApply": runtime_apply_refresh,
        }
        if abandonment_replacement:
            abandonment_intent = {
                "schema": "nexuscrm.test2-catalog-abandonment-intent/v1",
                "operationId": operation_id,
                "previousDesiredBomDigest": active_bom["desired_bom_sha256"],
                "replacementDesiredBomDigest": context.bom["desired_bom_sha256"],
                "reason": "replace-failed-unpublished-upgrade-with-signed-successor",
            }
            next_state["catalogAbandonment"] = {
                **abandonment_intent,
                "status": "pending",
                "evidenceDigest": sha256_bytes(canonical_bytes(abandonment_intent)),
            }
    pending = context.state_file.with_suffix(".json.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError(f"stale D3 Catalog successor state exists: {pending}")
    pending.write_bytes(canonical_bytes(next_state))
    os.chmod(pending, 0o600)
    os.replace(pending, context.state_file)
    return next_state


def apply_pending_catalog_abandonment(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> bool:
    """Abandon one unpublished upgrade with the signed successor Catalog image."""
    state = require_state(context, expected_phase="runtime-applied")
    receipt = state.get("catalogAbandonment")
    if receipt is None:
        return False
    required = {
        "schema", "operationId", "previousDesiredBomDigest",
        "replacementDesiredBomDigest", "reason", "status", "evidenceDigest",
    }
    if not isinstance(receipt, dict):
        raise DeployError("Catalog abandonment receipt is malformed")
    expected_keys = (
        required | {"output", "outputDigest"}
        if receipt.get("status") == "completed" else required
    )
    if set(receipt) != expected_keys:
        raise DeployError("Catalog abandonment receipt is malformed")
    operation_id = receipt.get("operationId")
    evidence_digest = receipt.get("evidenceDigest")
    if (
        receipt.get("schema") != "nexuscrm.test2-catalog-abandonment-intent/v1"
        or receipt.get("reason")
        != "replace-failed-unpublished-upgrade-with-signed-successor"
        or (
            receipt.get("status") == "pending"
            and receipt.get("replacementDesiredBomDigest")
            != context.bom["desired_bom_sha256"]
        )
        or not isinstance(receipt.get("previousDesiredBomDigest"), str)
        or not SHA256_RE.fullmatch(receipt["previousDesiredBomDigest"])
        or not isinstance(operation_id, str)
        or not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", operation_id)
        or not isinstance(evidence_digest, str)
        or not SHA256_RE.fullmatch(evidence_digest)
    ):
        raise DeployError("Catalog abandonment receipt disagrees with the signed successor")
    intent = {
        key: receipt[key]
        for key in (
            "schema", "operationId", "previousDesiredBomDigest",
            "replacementDesiredBomDigest", "reason",
        )
    }
    if sha256_bytes(canonical_bytes(intent)) != evidence_digest:
        raise DeployError("Catalog abandonment evidence digest changed")
    if receipt["status"] == "completed":
        output_path = Path(str(receipt["output"]))
        if (
            output_path.is_symlink() or not output_path.is_file()
            or sha256_bytes(output_path.read_bytes()) != receipt["outputDigest"]
        ):
            raise DeployError("completed Catalog abandonment lost its exact output")
        return False
    if receipt["status"] != "pending":
        raise DeployError("Catalog abandonment receipt has an unknown status")
    before = require_catalog_before(state)
    upgrade_authority = read_optional_unpublished_upgrade_authority(
        context.envv, operation_id,
    )
    if upgrade_authority is not None:
        authority_matches = upgrade_authority == before
    else:
        # The CLI may already have completed before the local receipt was
        # finalized.  Require its exact durable terminal receipt before the
        # restored active authority may prove the idempotent retry boundary.
        abandonment_completed = read_optional_catalog_abandonment_receipt(
            context.envv, operation_id, evidence_digest,
        )
        authority_matches = (
            abandonment_completed
            and read_optional_catalog_authority(context.envv) == before
        )
    if not authority_matches:
        raise DeployError("Catalog abandonment cannot prove the sealed active authority")

    output_dir = context.run_root / "catalog-abandonment"
    if output_dir.is_symlink() or (output_dir.exists() and not output_dir.is_dir()):
        raise DeployError("Catalog abandonment output path is not a regular directory")
    output_dir.mkdir(mode=0o700, exist_ok=True)
    output_path = output_dir / "catalog-abandonment.json"
    if output_path.is_symlink() or (
        output_path.exists() and not output_path.is_file()
    ):
        raise DeployError("Catalog abandonment output path is not a regular file")
    expected_output = {
        "schema": "nexuscrm.catalog-operation-abandonment/v1",
        "operation_id": operation_id,
        "state": "FAILED",
        "step": "ABANDONED",
        "evidence_digest": evidence_digest,
    }
    if not output_path.exists():
        private_dir = context.run_root / "private"
        if private_dir.is_symlink() or (
            private_dir.exists() and not private_dir.is_dir()
        ):
            raise DeployError("Catalog abandonment private path is not a regular directory")
        private_dir.mkdir(mode=0o700, exist_ok=True)
        db_env_file = private_dir / "catalog-abandonment-database.env"
        write_env_file(db_env_file, database_env(context.envv))
        container = container_name(context.run_id, "catalog-abandonment")
        execute((
            "docker", "run", "--rm", "--name", container,
            "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
            "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
            "--env-file", str(db_env_file),
            "--mount", f"type=bind,src={output_dir},dst=/output",
            context.images["catalog_reconcile"],
            "abandon-unpublished-upgrade",
            "-operation-id", operation_id,
            "-evidence-digest", evidence_digest,
            "-output", "/output/catalog-abandonment.json",
        ))
    output = load_json_object(output_path, "Catalog abandonment output")
    if output != expected_output:
        raise DeployError("Catalog abandonment output disagrees with its sealed intent")
    if read_optional_catalog_authority(context.envv) != before:
        raise DeployError("Catalog abandonment changed the active Catalog authority")
    os.chmod(output_path, 0o400)
    replace_state(
        context,
        expected_phase="runtime-applied",
        updates={
            "catalogAbandonment": {
                **receipt,
                "status": "completed",
                "output": str(output_path),
                "outputDigest": sha256_bytes(output_path.read_bytes()),
            },
        },
    )
    return True


def installation_org_id() -> str:
    """Return the explicit org identity for this Catalog installation.

    The incumbent CRM process does not persist a separate organization
    authority: its Salesforce-compatible identity response projects the
    product's fixed single-org identity.  The installer therefore accepts that
    identity directly instead of requiring the old HTTP process to be running.
    """
    org_id = required_env_value(os.environ, "D3_INSTALLATION_ORG_ID")
    if len(org_id.encode("utf-8")) > 255 or any(character.isspace() for character in org_id):
        raise DeployError("D3 installation org identity is not canonical")
    return org_id


def runtime_apply_recovery_pack_names(recovery: Mapping[str, Any]) -> list[str]:
    previous_reset = recovery.get("previousBaselineReset")
    raw_pack_names = (
        previous_reset.get("runtimePacks")
        if isinstance(previous_reset, dict)
        else None
    )
    if (
        not isinstance(raw_pack_names, list)
        or not raw_pack_names
        or any(not isinstance(name, str) or not name for name in raw_pack_names)
    ):
        raise DeployError("failed Runtime apply recovery has no sealed Runtime pack set")
    return sorted(set(raw_pack_names))


def runtime_apply_recovery_previous_epoch(recovery: Mapping[str, Any]) -> str:
    raw_epoch = recovery.get("acceptedPreviousParentTimestamp")
    if (
        not isinstance(raw_epoch, str)
        or not raw_epoch
        or raw_epoch.strip() != raw_epoch
        or len(raw_epoch.encode("utf-8")) > 255
        or any(character.isspace() for character in raw_epoch)
    ):
        raise DeployError("failed Runtime apply reset has no accepted reset epoch")
    return raw_epoch


def reset_test2_production_baseline(context: D3Context) -> dict[str, Any]:
    """Reset test2 once to its production-parent snapshot before D3 mutation."""
    state = require_state(context)
    if isinstance(state.get("baselineReset"), dict):
        return dict(state)
    phase = str(state.get("phase", ""))
    if phase not in BASELINE_RESET_ELIGIBLE_PHASES:
        raise DeployError(
            "D3 production-baseline reset is allowed only before Catalog mutation; "
            f"current phase is {phase or '<empty>'}"
        )
    if phase == "baseline-reset-in-progress":
        raise DeployError(
            "D3 production-baseline reset has an uncertain provider outcome; "
            "refusing a destructive retry. Recover test2 explicitly, abandon this "
            "attempt, and start a new clean D3 run"
        )
    if owned_containers():
        raise DeployError("D3 production-baseline reset refuses to run while D3 containers exist")
    test_deploy.require_test_clone_scrub_target_identity(
        dict(context.envv),
        subject="D3 production-baseline reset",
    )
    runtime_apply_recovery = state.get("runtimeApplyRecovery")
    recovery_pack_names: list[str] | None = None
    if runtime_apply_recovery is not None and not isinstance(runtime_apply_recovery, dict):
        raise DeployError("failed Runtime apply recovery receipt is malformed")
    if isinstance(runtime_apply_recovery, dict):
        recovery_pack_names = runtime_apply_recovery_pack_names(runtime_apply_recovery)
        start = require_catalog_start(state)
        expected_catalog = (
            require_catalog_before(state) if start["mode"] == "UPGRADE" else None
        )
        if read_optional_catalog_authority(context.envv) != expected_catalog:
            raise DeployError(
                "failed Runtime apply recovery refuses changed Catalog authority"
            )
    reset_updates: dict[str, Any] = {
        "phase": "baseline-reset-in-progress",
        "baselineResetFromPhase": phase,
    }
    if isinstance(runtime_apply_recovery, dict):
        branch_id = required_env_value(context.envv, "TIDB_CLOUD_BRANCH_ID")
        previous_parent_timestamp = tidb_branch_parent_timestamp(
            tidb_cloud_api(context.envv, "GET", f"/branches/{branch_id}")
        )
        runtime_apply_recovery = dict(runtime_apply_recovery)
        runtime_apply_recovery["acceptedPreviousParentTimestamp"] = (
            previous_parent_timestamp
        )
        reset_updates["runtimeApplyRecovery"] = runtime_apply_recovery
    state = replace_state(
        context,
        expected_phase=phase,
        updates=reset_updates,
    )
    org_id = state.get("baselineResetInstallationOrgId")
    if not isinstance(org_id, str) or not org_id:
        org_id = installation_org_id()
        state = replace_state(
            context,
            expected_phase="baseline-reset-in-progress",
            updates={"baselineResetInstallationOrgId": org_id},
        )

    # The branch reset replaces the database under the fixed test2 route. Stop
    # every previous writer and disable autostart before the provider operation.
    staging.quiesce_staging_target_runtime(
        dict(context.envv),
        runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    reset_summary = test_deploy.run_test_reset_branch_snapshot_stage(
        context.env_file,
        "test2",
    )
    fresh_env = load_env_file(context.env_file)
    test_deploy.require_test_clone_scrub_target_identity(
        fresh_env,
        subject="D3 post-reset recovery",
    )
    runtime_apply_recovery = state.get("runtimeApplyRecovery")
    if isinstance(runtime_apply_recovery, dict):
        start = require_catalog_start(state)
        expected_catalog = (
            require_catalog_before(state) if start["mode"] == "UPGRADE" else None
        )
        if read_optional_catalog_authority(fresh_env) != expected_catalog:
            raise DeployError(
                "failed Runtime cleanup reset refuses changed Catalog authority"
            )
        if recovery_pack_names is None:
            raise DeployError("failed Runtime apply recovery lost its sealed Runtime pack set")
        pack_names = recovery_pack_names
    else:
        exact_sources = state.get("exactSources")
        if not isinstance(exact_sources, dict):
            raise DeployError("D3 production-baseline reset has no exact L2 source")
        l2_root = Path(str(exact_sources.get("l2Root", ""))).resolve()
        if not l2_root.is_dir():
            raise DeployError("D3 production-baseline reset lost the exact L2 source")
        pack_names = runtime_pack_names(l2_root)
    if not pack_names:
        raise DeployError("D3 production-baseline reset resolved no Runtime packs")
    purged = purge_reset_snapshot_connector_queue(fresh_env)
    reset_drained_workflow_action_syncstate_for_packs(fresh_env, pack_names)
    update_env_file(
        context.env_file,
        {test_deploy.RESET_SNAPSHOT_CONNECTOR_QUEUE_HYGIENE_PENDING: "0"},
    )
    baseline_reset = {
        "source": "prod-tidb-reset-snapshot",
        "targetSlot": "test2",
        "providerResetEpoch": reset_summary["provider_reset_epoch"],
        "resetHttpStatus": reset_summary["reset_http_status"],
        "resetRetryHttpStatus": reset_summary["reset_retry_http_status"],
        "providerCredentialSecretScrub": reset_summary[
            "provider_credential_secret_scrub"
        ],
        # The Salesforce-compatible org identity is immutable across a TiDB
        # branch reset. Seal it while the incumbent process is still serving so
        # bootstrap never needs to restart or consult legacy after reset.
        "legacyOrgId": org_id,
        **reset_snapshot_connector_queue_purge_summary(purged),
        "runtimePacks": sorted(pack_names),
    }
    return replace_state(
        context,
        expected_phase="baseline-reset-in-progress",
        updates={
            "phase": "baseline-reset",
            "baselineReset": baseline_reset,
            "catalogStart": None,
            "catalogBefore": None,
            "workerOwnershipTableBefore": None,
            "workerOwnershipBefore": None,
            "runtimeApply": None,
            "runtimeAppShell": None,
        },
    )


def preserve_current_test2_database(context: D3Context) -> dict[str, Any]:
    """Seal the current Test2 database as the stopped-upgrade input.

    Test2 already carries the accepted production-derived snapshot.  A normal
    independent-process deploy must not reset it again: it records the current
    database identity, then the existing quiesce step establishes the single-
    writer boundary before any setup or Catalog mutation.
    """
    state = require_state(context, expected_phase="artifacts-ready")
    test_deploy.require_test_clone_scrub_target_identity(
        dict(context.envv),
        subject="D3 retained Test2 database",
    )
    if owned_containers():
        raise DeployError("current-database adoption refuses active D3 containers")
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, dict):
        raise DeployError("current-database adoption has no exact L2 source")
    l2_root = Path(str(exact_sources.get("l2Root", ""))).resolve()
    if not l2_root.is_dir():
        raise DeployError("current-database adoption lost the exact L2 source")
    pack_names = runtime_pack_names(l2_root)
    if not pack_names:
        raise DeployError("current-database adoption resolved no Runtime packs")

    baseline: dict[str, Any] = {
        "source": "retained-test2-database",
        "targetSlot": "test2",
        "resetPerformed": False,
        "runtimePacks": sorted(pack_names),
    }
    if not catalog_authority_tables_present(context.envv):
        baseline["installationOrgId"] = installation_org_id()
    return replace_state(
        context,
        expected_phase="artifacts-ready",
        updates={
            "phase": "baseline-reset",
            "baselineReset": baseline,
            "catalogStart": None,
            "catalogBefore": None,
            "workerOwnershipTableBefore": None,
            "workerOwnershipBefore": None,
            "runtimeApply": None,
            "runtimeAppShell": None,
        },
    )


def quiesce(context: D3Context) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={"baseline-reset", "quiesce-in-progress"},
    )
    if owned_containers():
        raise DeployError("D3-owned containers already exist without a serving state")
    preflight_runtime_filesystem_storage_mounts(
        context,
        runtime_image=context.images["runtime_api"],
    )
    if state["phase"] == "baseline-reset":
        if catalog_authority_tables_present(context.envv):
            # This is only a pre-quiescence safety check. The authoritative
            # snapshot is read again after all prior writers are stopped.
            # Rejecting an empty/malformed installation here avoids taking the
            # legacy tuple offline for an upgrade that cannot start.
            read_catalog_authority(context.envv)
            start = {"mode": "UPGRADE"}
        else:
            baseline_reset = state.get("baselineReset")
            org_id = (
                baseline_reset.get("installationOrgId")
                if isinstance(baseline_reset, dict)
                else None
            )
            if not isinstance(org_id, str) or not org_id:
                raise DeployError("D3 baseline has no sealed installation org identity")
            start = {
                "mode": "PREPARE",
                "orgId": org_id,
                "runtimeAudience": "nexuscrm-runtime-api",
                "expectedCatalogRevision": 0,
                "expectedSecurityEpoch": 0,
                "schemaGeneration": 1,
                "engineGeneration": 1,
            }
        state = replace_state(
            context,
            expected_phase="baseline-reset",
            updates={"phase": "quiesce-in-progress", "catalogStart": start},
        )
    staging.quiesce_staging_target_runtime(
        dict(context.envv),
        runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    baseline_reset = state.get("baselineReset")
    if not isinstance(baseline_reset, dict):
        raise DeployError("quiescing D3 state lost its baseline receipt")
    baseline_reset = dict(baseline_reset)
    start = state.get("catalogStart")
    if not isinstance(start, dict):
        raise DeployError("quiescing D3 state has no Catalog start authority")
    updates: dict[str, Any] = {
        "phase": "quiesced",
        "baselineReset": baseline_reset,
    }
    if start.get("mode") == "UPGRADE":
        # Read mutable authority only after all prior writers are durably stopped.
        authority = read_catalog_authority(context.envv)
        updates["catalogStart"] = {
            "mode": "UPGRADE",
            "refreshActiveRelease": (
                authority.active_release_digest
                == context.bom["app_installation"]["app_release_digest"]
            ),
        }
        updates["catalogBefore"] = {
            "appInstallationId": authority.app_installation_id,
            "orgId": authority.org_id,
            "activeReleaseDigest": authority.active_release_digest,
            "catalogRevision": authority.catalog_revision,
            "runtimeAudience": authority.runtime_audience,
            "securityEpoch": authority.security_epoch,
            "activePublicationDigest": authority.active_publication_digest,
            "schemaGeneration": authority.schema_generation,
            "engineGeneration": authority.engine_generation,
        }
    elif start.get("mode") == "PREPARE":
        if read_optional_catalog_authority(context.envv) is not None:
            raise DeployError("Catalog authority appeared during D3 bootstrap quiescence")
    else:
        raise DeployError("D3 Catalog start mode is unknown")
    worker_table_present, worker_ownership = read_optional_worker_ownership(context.envv)
    updates["workerOwnershipTableBefore"] = worker_table_present
    updates["workerOwnershipBefore"] = worker_ownership_payload(worker_ownership)
    next_state = replace_state(
        context,
        expected_phase="quiesce-in-progress",
        updates=updates,
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "quiesced",
        "runId": context.run_id,
        "resolvedBomFileDigest": context.bom_file_digest,
        "legacyWriterProcesses": 0,
        "catalogStart": next_state["catalogStart"],
        "catalogBefore": next_state.get("catalogBefore"),
        "workerOwnershipBefore": next_state["workerOwnershipBefore"],
    }


DATABASE_ENV_KEYS = (
    "TIDB_HOST", "TIDB_PORT", "TIDB_USER", "TIDB_PASSWORD", "TIDB_DATABASE",
    "TIDB_TLS_DISABLE",
)

RUNTIME_STORAGE_ENV_KEYS = (
    "FILE_STORAGE_PROVIDER", "FILE_STORAGE_ROOT", "FILE_STORAGE_LEGACY_UPLOAD_ROOT",
    "FILE_STORAGE_BUCKET", "FILE_STORAGE_REGION", "FILE_STORAGE_ENDPOINT",
    "FILE_STORAGE_ACCESS_KEY_ID", "FILE_STORAGE_SECRET_ACCESS_KEY",
    "FILE_STORAGE_SESSION_TOKEN", "FILE_STORAGE_PREFIX", "FILE_STORAGE_USE_PATH_STYLE",
    "FILE_STORAGE_ADDITIONAL_S3_AUTHORITIES_JSON",
    "NEXUSCRM_RUNTIME_ROOT",
)

D3_FEDERATED_OIDC_SCHEMA = "nexuscrm.serviceification-d3-federated-oidc/v1"
RUNTIME_WORKLOAD_IDENTITIES_SCHEMA = "nexuscrm.runtime-workload-identities/v1"
RUNTIME_SIGNING_KEYS_SCHEMA = "nexuscrm.runtime-signing-keys/v1"
RUNTIME_SERVER_NAME = "runtime-api.d3.test2.local"
DEFAULT_RUNTIME_API_ADMISSION_LIMIT = 16
RUNTIME_API_ADMISSION_LIMIT = DEFAULT_RUNTIME_API_ADMISSION_LIMIT


def write_env_file(path: Path, values: Mapping[str, str]) -> None:
    normalized: list[str] = []
    for key, value in values.items():
        if not re.fullmatch(r"[A-Z][A-Z0-9_]*", key):
            raise DeployError(f"invalid container environment key: {key}")
        if "\x00" in value or "\n" in value or "\r" in value:
            raise DeployError(f"container environment value for {key} contains a line break")
        normalized.append(f"{key}={value}")
    path.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
    path.write_text("\n".join(normalized) + "\n")
    os.chmod(path, 0o600)


def database_env(envv: Mapping[str, str]) -> dict[str, str]:
    required = {"TIDB_HOST", "TIDB_PORT", "TIDB_USER", "TIDB_DATABASE"}
    missing = sorted(key for key in required if not envv.get(key))
    if missing:
        raise DeployError("test2 database authority is incomplete: " + ", ".join(missing))
    return {
        **{key: str(envv[key]) for key in DATABASE_ENV_KEYS if key in envv},
        "NEXUSCRM_DB_MAX_OPEN_CONNS": "4",
        "NEXUSCRM_DB_MAX_IDLE_CONNS": "2",
    }


def runtime_storage_env(envv: Mapping[str, str]) -> dict[str, str]:
    # D3 filesystem writes use a new Test-only authority. The retained offline
    # tree can accompany a database refresh and is therefore never mounted,
    # traversed, or mutated by the serviceified runtime.
    provider = str(envv.get("FILE_STORAGE_PROVIDER", "filesystem")).strip()
    result = {
        key: str(envv[key]) for key in RUNTIME_STORAGE_ENV_KEYS
        if key in envv and str(envv[key]) != ""
    }
    result["FILE_STORAGE_PROVIDER"] = provider
    if "FILE_STORAGE_ADDITIONAL_S3_AUTHORITIES_JSON" not in result:
        # A Test2 database refreshed from Prod contains source-scoped Salesforce
        # history rows.  Preserve Test2's own upload destination while admitting
        # the one L2-owned, read-only archive authority that owns those bytes.
        result["FILE_STORAGE_ADDITIONAL_S3_AUTHORITIES_JSON"] = json.dumps(
            [{
                "name": "salesforce-history",
                "source_type": "salesforce",
                **salesforce_history_archive_authority(),
            }],
            separators=(",", ":"),
            sort_keys=True,
        )
    if provider == "filesystem":
        env_root = str(envv.get("ENV_ROOT", "")).strip()
        if not env_root or not Path(env_root).expanduser().is_absolute():
            raise DeployError("D3 Test filesystem storage requires an absolute ENV_ROOT")
        result["FILE_STORAGE_ROOT"] = str(d3_filesystem_storage_root(Path(env_root)))
    elif provider == "s3":
        required = ("FILE_STORAGE_BUCKET", "FILE_STORAGE_REGION")
        missing = [key for key in required if not result.get(key)]
        if missing:
            raise DeployError("test2 S3 storage authority is incomplete: " + ", ".join(missing))
    else:
        raise DeployError(f"test2 file-storage provider is unsupported: {provider}")
    return result


def d3_filesystem_storage_root(env_root: Path) -> Path:
    """Return the one D3 filesystem authority without touching its descendants."""
    expanded = env_root.expanduser()
    if not expanded.is_absolute():
        raise DeployError("D3 Test filesystem storage requires an absolute ENV_ROOT")
    try:
        resolved_env_root = expanded.resolve(strict=True)
    except OSError as exc:
        raise DeployError("D3 Test ENV_ROOT is unavailable") from exc
    if not resolved_env_root.is_dir():
        raise DeployError("D3 Test ENV_ROOT is unavailable")
    return resolved_env_root / "shared" / "data" / D3_FILESYSTEM_STORAGE_DIRECTORY


def open_or_create_directory_no_symlink(
    parent_fd: int,
    name: str,
) -> tuple[int, bool]:
    """Open one child directory without following a symlink at that boundary."""
    created = False
    try:
        os.mkdir(name, mode=0o700, dir_fd=parent_fd)
        created = True
    except FileExistsError:
        pass
    try:
        child_fd = os.open(
            name,
            os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
            dir_fd=parent_fd,
        )
    except OSError as exc:
        raise DeployError(
            f"D3 Test filesystem storage path component is unsafe: {name}"
        ) from exc
    return child_fd, created


def ensure_isolated_runtime_filesystem_storage(
    context: D3Context,
    storage: Mapping[str, str],
) -> Path:
    """Provision or verify the sealed Test-only D3 filesystem authority."""
    expected = d3_filesystem_storage_root(context.env_root)
    configured = Path(str(storage.get("FILE_STORAGE_ROOT", ""))).expanduser()
    if configured != expected:
        raise DeployError("D3 Test filesystem storage root is not the isolated authority")

    directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
    try:
        current_fd = os.open(str(expected.parents[2]), directory_flags)
    except OSError as exc:
        raise DeployError("D3 Test ENV_ROOT is unavailable") from exc
    opened_fds = [current_fd]
    created_storage = False
    try:
        for component in ("shared", "data", D3_FILESYSTEM_STORAGE_DIRECTORY):
            current_fd, created = open_or_create_directory_no_symlink(
                current_fd,
                component,
            )
            opened_fds.append(current_fd)
            if component == D3_FILESYSTEM_STORAGE_DIRECTORY:
                created_storage = created

        storage_stat = os.fstat(current_fd)
        if storage_stat.st_uid != os.getuid() or storage_stat.st_gid != os.getgid():
            raise DeployError(
                "D3 Test filesystem storage is not owned by the Runtime process identity"
            )

        authority = {
            "schema": D3_FILESYSTEM_STORAGE_MARKER_SCHEMA,
            "environment": str(context.envv.get("ENV_NAME", "")).strip(),
        }
        marker_flags = os.O_NOFOLLOW
        if created_storage:
            marker_fd = os.open(
                D3_FILESYSTEM_STORAGE_MARKER,
                os.O_WRONLY | os.O_CREAT | os.O_EXCL | marker_flags,
                0o600,
                dir_fd=current_fd,
            )
            with os.fdopen(marker_fd, "wb") as marker_file:
                marker_file.write(canonical_bytes(authority))
                marker_file.flush()
                os.fsync(marker_file.fileno())
        else:
            try:
                marker_fd = os.open(
                    D3_FILESYSTEM_STORAGE_MARKER,
                    os.O_RDONLY | marker_flags,
                    dir_fd=current_fd,
                )
            except OSError as exc:
                raise DeployError(
                    "D3 Test filesystem storage predates the isolated authority marker"
                ) from exc
            with os.fdopen(marker_fd, "rb") as marker_file:
                marker_stat = os.fstat(marker_file.fileno())
                if not stat.S_ISREG(marker_stat.st_mode):
                    raise DeployError("D3 Test filesystem storage marker is unreadable")
                try:
                    actual = json.loads(marker_file.read())
                except json.JSONDecodeError as exc:
                    raise DeployError(
                        "D3 Test filesystem storage marker is unreadable"
                    ) from exc
            if actual != authority:
                raise DeployError("D3 Test filesystem storage marker changed authority")

        os.fchmod(current_fd, 0o700)
    finally:
        for opened_fd in reversed(opened_fds):
            os.close(opened_fd)
    return configured


def worker_environment_values(context: D3Context, stem: str) -> dict[str, str]:
    if stem not in context.worker_stems:
        raise DeployError(f"unknown Runtime Worker role: {stem}")
    values = {
        **database_env(context.envv),
        "NEXUSCRM_RUNTIME_WORKER_ADMIN_ADDR": (
            f"127.0.0.1:{context.ports.worker_admin[stem]}"
        ),
        "NEXUSCRM_RUNTIME_WORKLOAD_IDENTITY_TOKEN_FILE": "/run/secrets/workload.token",
        "NEXUSCRM_RUNTIME_WORKER_CONFIG_FILE": "/etc/nexuscrm/worker-role.json",
    }
    if stem in OBJECT_STORAGE_WORKER_STEMS:
        values.update(runtime_storage_env(context.envv))
    if stem == "record-event-dispatch":
        values["NEXUSCRM_RUNTIME_WORKER_TIMED_FLOW_OWNER_CONFIG_FILE"] = (
            "/etc/nexuscrm/scheduled-flow-owner.json"
        )
    return values


RESET_SNAPSHOT_BASELINE_SOURCES = {
    "prod-tidb-reset-snapshot",
    "prod-tidb-reset-snapshot-recovered",
}


def d3_federated_oidc_document(envv: Mapping[str, str]) -> dict[str, Any] | None:
    methods = {
        value.strip().lower()
        for value in str(envv.get("NEXUS_LOGIN_METHODS", "")).split(",")
        if value.strip()
    }
    client_id = str(envv.get("GOOGLE_CLIENT_ID", ""))
    client_secret = str(envv.get("GOOGLE_CLIENT_SECRET", ""))
    wants_google = "google" in methods or bool(client_id) or bool(client_secret)
    if not wants_google:
        return None
    if (
        not client_id or not client_secret
        or client_id.strip() != client_id or client_secret.strip() != client_secret
    ):
        raise DeployError(
            "test2 Google session control requires canonical GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET"
        )
    return {
        "schema": D3_FEDERATED_OIDC_SCHEMA,
        "providers": [{
            "provider_id": "google",
            "audience": client_id,
            "session_scope": "api",
            "trusted_issuers": ["accounts.google.com", "https://accounts.google.com"],
            "jwks_url": "https://www.googleapis.com/oauth2/v3/certs",
        }],
    }


def write_protected_json(path: Path, value: Mapping[str, Any]) -> None:
    path.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
    if path.exists() or path.is_symlink():
        raise DeployError(f"refusing to replace serving authority input: {path}")
    path.write_bytes(canonical_bytes(dict(value)))
    os.chmod(path, 0o400)


def load_json_object(path: Path, label: str) -> dict[str, Any]:
    if not path.is_file() or path.is_symlink():
        raise DeployError(f"{label} is not a regular file: {path}")
    try:
        value = json.loads(path.read_text())
    except (OSError, json.JSONDecodeError) as exc:
        raise DeployError(f"{label} is not canonical JSON: {path}") from exc
    if not isinstance(value, dict):
        raise DeployError(f"{label} is not a JSON object: {path}")
    return value


def signed_l2_programmatic_oauth_authority(
    context: D3Context,
    state: Mapping[str, Any],
    *,
    label: str = "D3 programmatic OAuth",
) -> tuple[Path, str, str]:
    """Resolve the password-grant client from the signed exact L2 metadata tree."""
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, Mapping):
        raise DeployError(f"{label} has no exact L2 source")
    metadata = context.bom.get("l2_metadata")
    if not isinstance(metadata, Mapping):
        raise DeployError(f"{label} has no signed L2 metadata")

    state_l2_commit = str(exact_sources.get("l2Commit", "")).strip()
    bom_l2_commit = str(metadata.get("commit", "")).strip()
    if re.fullmatch(r"[0-9a-f]{40}", state_l2_commit) is None:
        raise DeployError(f"{label} has an invalid state L2 commit")
    if re.fullmatch(r"[0-9a-f]{40}", bom_l2_commit) is None:
        raise DeployError(f"{label} has an invalid resolved BOM L2 commit")
    if state_l2_commit != bom_l2_commit:
        raise DeployError(
            f"{label} L2 commit differs from the resolved BOM"
        )

    l2_root_text = str(exact_sources.get("l2Root", "")).strip()
    if not l2_root_text:
        raise DeployError(f"{label} has an empty L2 source")
    l2_root = require_exact_checkout(
        Path(l2_root_text),
        state_l2_commit,
        "L2 metadata",
    )

    pack_path = Path(str(metadata.get("path", "")))
    if pack_path.is_absolute() or not pack_path.parts or ".." in pack_path.parts:
        raise DeployError(f"{label} has an invalid signed L2 path")
    pack_root = (l2_root / pack_path).resolve()
    try:
        pack_root.relative_to(l2_root)
    except ValueError as exc:
        raise DeployError(
            f"{label} L2 path escapes its exact checkout"
        ) from exc

    expected_tree = str(metadata.get("tree_sha256", "")).strip()
    try:
        actual_tree = metadata_tree_sha256(pack_root)
    except FinalPackContractError as exc:
        raise DeployError(
            f"{label} cannot read the signed L2 tree"
        ) from exc
    if actual_tree != expected_tree:
        raise DeployError(
            f"{label} L2 tree differs from the signed BOM"
        )

    client = load_json_object(
        pack_root / "connected_apps" / "salesforce_compat_client.json",
        f"{label} programmatic OAuth client",
    )
    try:
        client_id, client_secret = validate_programmatic_oauth_client(client)
    except SystemExit as exc:
        raise DeployError(
            f"{label} programmatic OAuth client is invalid: {exc}"
        ) from exc
    return l2_root, client_id, client_secret


def materialize_connected_mail_definition(
    state: Mapping[str, Any], target: Path,
) -> Path:
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, Mapping):
        raise DeployError("connected-mail activation has no exact AppPacks source")
    app_packs_root = Path(str(exact_sources.get("appPacksRoot", ""))).resolve()
    source = app_packs_root / CONNECTED_MAIL_PROVIDER_PATH
    manifest_path = app_packs_root / "gmail-connected-mail-connector" / "pack.json"
    if (
        source.is_symlink()
        or not source.is_file()
        or manifest_path.is_symlink()
        or not manifest_path.is_file()
    ):
        raise DeployError("connected-mail provider definition is unavailable")
    manifest = load_json_object(manifest_path, "Gmail connected-mail connector manifest")
    extensions = manifest.get("extensions")
    definitions = (
        extensions.get("connectedMailProviderDefinitions")
        if isinstance(extensions, dict) else None
    )
    matches = [
        definition for definition in definitions or []
        if isinstance(definition, dict)
        and definition.get("connectorKey") == "gmail-connected-mail"
        and definition.get("path") == "integrations/connected_mail_gmail.json"
    ]
    if len(matches) != 1 or set(matches[0]) != {"connectorKey", "path", "sha256"}:
        raise DeployError("Gmail connector has no exact connected-mail definition authority")
    raw = source.read_bytes()
    digest = hashlib.sha256(raw).hexdigest()
    if matches[0].get("sha256") != digest:
        raise DeployError("Gmail connected-mail definition disagrees with its pack manifest")
    target.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
    if target.exists() or target.is_symlink():
        raise DeployError("refusing to replace connected-mail provider authority")
    target.write_bytes(raw)
    os.chmod(target, 0o400)
    return target


def provision_token(path: Path) -> str:
    path.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
    if path.exists() or path.is_symlink():
        raise DeployError(f"refusing to rotate an existing workload identity: {path}")
    token = secrets.token_hex(32)
    path.write_text(token + "\n")
    os.chmod(path, 0o400)
    return token


def provision_workload_identities(root: Path, *, mcp: bool = False) -> dict[str, str]:
    tokens = {
        "integration-service": provision_token(root / "integration.token"),
        "sfcrm-app-service": provision_token(root / "sfcrm.token"),
    }
    if mcp:
        tokens["mcp-worker"] = provision_token(root / "mcp.token")
    identities = {
        "schema": RUNTIME_WORKLOAD_IDENTITIES_SCHEMA,
        "identities": [
            {"workloadId": workload_id, "token": tokens[workload_id]}
            for workload_id in sorted(tokens)
        ],
    }
    write_protected_json(root / "runtime-workload-identities.json", identities)
    return tokens


def provision_runtime_signing_key(
    root: Path,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_bytes: Callable[[Sequence[str]], bytes] | None = None,
) -> Path:
    root.mkdir(parents=True, mode=0o700, exist_ok=True)
    pem = root / "runtime-signing.pem"
    output = root / "runtime-signing-keys.json"
    if any(path.exists() or path.is_symlink() for path in (pem, output)):
        raise DeployError("refusing to rotate an existing Runtime signing identity")
    execute(("openssl", "genpkey", "-algorithm", "ED25519", "-out", str(pem)))
    if capture_bytes is None:
        def capture_bytes(args: Sequence[str]) -> bytes:
            return subprocess.run(list(args), check=True, capture_output=True).stdout
    private_der = capture_bytes(("openssl", "pkey", "-in", str(pem), "-outform", "DER"))
    public_der = capture_bytes((
        "openssl", "pkey", "-in", str(pem), "-pubout", "-outform", "DER",
    ))
    if len(private_der) < 32 or len(public_der) < 32:
        raise DeployError("OpenSSL emitted an invalid Ed25519 signing identity")
    encoded = base64.b64encode(private_der[-32:] + public_der[-32:]).decode("ascii")
    write_protected_json(output, {
        "schema": RUNTIME_SIGNING_KEYS_SCHEMA,
        "active": {
            "signing_key_id": f"d3-test2-{root.parent.parent.name}",
            "private_key_base64": encoded,
        },
        "overlap": [],
    })
    os.chmod(pem, 0o400)
    return output


def provision_tls_identity(
    root: Path,
    server_name: str,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> tuple[Path, Path]:
    root.mkdir(parents=True, mode=0o700, exist_ok=True)
    certificate = root / "server.crt"
    private_key = root / "server.key"
    if any(path.exists() or path.is_symlink() for path in (certificate, private_key)):
        raise DeployError("refusing to rotate an existing Runtime TLS identity")
    execute((
        "openssl", "req", "-x509", "-newkey", "rsa:2048", "-sha256", "-nodes",
        "-days", str(D3_SERVING_TLS_VALIDITY_DAYS), "-subj", f"/CN={server_name}",
        "-addext", f"subjectAltName=DNS:{server_name}",
        "-keyout", str(private_key), "-out", str(certificate),
    ))
    os.chmod(certificate, 0o400)
    os.chmod(private_key, 0o400)
    return certificate, private_key


def required_env_value(envv: Mapping[str, str], name: str) -> str:
    value = str(envv.get(name, ""))
    if not value or value.strip() != value or "\x00" in value or "\n" in value or "\r" in value:
        raise DeployError(f"test2 runtime authority is missing canonical {name}")
    return value


def optional_env_values(envv: Mapping[str, str], names: Sequence[str]) -> dict[str, str]:
    result: dict[str, str] = {}
    for name in names:
        value = str(envv.get(name, ""))
        if not value:
            continue
        if value.strip() != value or "\x00" in value or "\n" in value or "\r" in value:
            raise DeployError(f"test2 runtime authority has non-canonical {name}")
        result[name] = value
    return result


def connected_mail_oauth_env(envv: Mapping[str, str]) -> dict[str, str]:
    client_id = str(envv.get(CONNECTED_MAIL_CLIENT_ID_ENV, ""))
    client_secret = str(envv.get(CONNECTED_MAIL_CLIENT_SECRET_ENV, ""))
    if not client_id and not client_secret:
        return {}
    if not client_id or not client_secret:
        raise DeployError("connected-mail OAuth client ID and secret must be supplied together")
    for name, value in (
        (CONNECTED_MAIL_CLIENT_ID_ENV, client_id),
        (CONNECTED_MAIL_CLIENT_SECRET_ENV, client_secret),
    ):
        if value.strip() != value or "\x00" in value or "\n" in value or "\r" in value:
            raise DeployError(f"connected-mail OAuth authority has non-canonical {name}")
    return {
        CONNECTED_MAIL_CLIENT_ID_ENV: client_id,
        CONNECTED_MAIL_CLIENT_SECRET_ENV: client_secret,
    }


def validate_public_origin(value: str, *, allow_loopback_http: bool = False) -> str:
    public_origin = value.rstrip("/")
    parsed_origin = urllib.parse.urlsplit(public_origin)
    is_https = parsed_origin.scheme == "https"
    is_local_http = (
        allow_loopback_http
        and parsed_origin.scheme == "http"
        and parsed_origin.hostname in {"127.0.0.1", "localhost", "::1"}
        and parsed_origin.username is None
        and parsed_origin.password is None
        and parsed_origin.path in {"", "/"}
        and not parsed_origin.query
        and not parsed_origin.fragment
    )
    if not (is_https or is_local_http):
        raise DeployError(
            "PUBLIC_BASE_URL must be canonical HTTPS; local clean-install may use "
            "an explicit loopback HTTP origin"
        )
    return public_origin


def verify_catalog_after(context: D3Context, state: Mapping[str, Any]) -> CatalogAuthority:
    after = state.get("catalogAfter")
    if not isinstance(after, dict):
        raise DeployError("catalog-applied state has no exact post-upgrade authority")
    current = read_catalog_authority(context.envv)
    expected_release = context.bom["app_installation"]["app_release_digest"]
    if (
        current.app_installation_id != after.get("appInstallationId")
        or current.org_id != after.get("orgId")
        or current.active_release_digest != expected_release
        or current.catalog_revision != after.get("catalogRevision")
        or current.runtime_audience != after.get("runtimeAudience")
        or current.security_epoch != after.get("securityEpoch")
        or current.active_publication_digest != after.get("activePublicationDigest")
        or current.schema_generation != after.get("schemaGeneration")
        or current.engine_generation != after.get("engineGeneration")
    ):
        raise DeployError("Catalog authority no longer matches the exact D3 reconcile receipt")
    return current


def validate_governed_reference_data_evidence(
    evidence: Mapping[str, Any],
) -> tuple[list[str], int]:
    invocations = evidence.get("invocations")
    if (
        evidence.get("status") != "passed"
        or evidence.get("profile") != "test"
        or evidence.get("phase") != "governed_reference_data"
        or evidence.get("execution_mode") != "apply"
        or not isinstance(invocations, list)
    ):
        raise DeployError("governed reference-data apply evidence is incomplete")

    invocation_by_id = {
        invocation.get("invocation_id"): invocation
        for invocation in invocations
        if isinstance(invocation, dict)
        and isinstance(invocation.get("invocation_id"), str)
    }
    if len(invocation_by_id) != len(invocations):
        raise DeployError("governed reference-data invocation evidence is incomplete")

    reference_invocation = invocation_by_id.get(
        "test.pack_reference_data.governed_reference_data"
    )
    summary = (
        reference_invocation.get("summary")
        if isinstance(reference_invocation, dict)
        else None
    )
    selection = (
        reference_invocation.get("selection")
        if isinstance(reference_invocation, dict)
        else None
    )
    selected_objects = selection.get("objects") if isinstance(selection, dict) else None
    if (
        not isinstance(reference_invocation, dict)
        or reference_invocation.get("module_id") != "pack_reference_data"
        or reference_invocation.get("status") != "applied"
        or not isinstance(selected_objects, list)
        or not selected_objects
        or any(not isinstance(item, str) or not item for item in selected_objects)
        or not isinstance(summary, dict)
        or summary.get("coverage_complete") is not True
        or summary.get("reference_data_objects") != selected_objects
        or not isinstance(summary.get("reference_data_record_count"), int)
        or isinstance(summary.get("reference_data_record_count"), bool)
        or summary["reference_data_record_count"] <= 0
    ):
        raise DeployError("governed reference-data adapter evidence is incomplete")

    permissions_invocation = invocation_by_id.get(
        "test.credits_permissions.governed_reference_data"
    )
    permissions_summary = (
        permissions_invocation.get("summary")
        if isinstance(permissions_invocation, dict)
        else None
    )
    expected_permission_counts = {
        "permission_sets": 5,
        "object_permissions": 5,
        "field_permissions": 29,
    }
    if (
        not isinstance(permissions_invocation, dict)
        or permissions_invocation.get("module_id") != "credits_permissions_seed"
        or permissions_invocation.get("status") != "applied"
        or permissions_invocation.get("selection") != {}
        or not isinstance(permissions_summary, dict)
        or permissions_summary.get("schema_version") != 1
        or permissions_summary.get("format") != "deploy-seed-script-v1"
        or permissions_summary.get("module_id") != "credits_permissions_seed"
        or permissions_summary.get("dry_run") is not False
        or permissions_summary.get("apply_engine") != "direct_sql"
        or permissions_summary.get("planned_counts") != expected_permission_counts
        or permissions_summary.get("applied_counts") != expected_permission_counts
        or permissions_summary.get("coverage_complete") is not True
        or permissions_summary.get("verification_status") != "skipped"
    ):
        raise DeployError("Credits permission-definition evidence is incomplete")

    if set(invocation_by_id) != {
        "test.pack_reference_data.governed_reference_data",
        "test.credits_permissions.governed_reference_data",
    }:
        raise DeployError("governed reference-data invocation evidence is incomplete")

    return selected_objects, summary["reference_data_record_count"]


def apply_governed_reference_data(
    context: D3Context,
    *,
    apply_seed: Callable[..., dict[str, object]] = offline.apply_deploy_seed_phase,
) -> dict[str, Any]:
    """Reconcile the exact L2-owned Test2 reference data before serving starts."""
    state = require_state(
        context,
        expected_phase={
            "catalog-applied", "governed-reference-data-in-progress",
        },
    )
    verify_catalog_after(context, state)
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, dict):
        raise DeployError("governed reference-data apply has no exact L2 source")
    l2_commit = str(exact_sources.get("l2Commit", ""))
    l2_root = require_exact_checkout(
        Path(str(exact_sources.get("l2Root", ""))),
        l2_commit,
        "L2 metadata",
    )
    if state["phase"] == "catalog-applied":
        state = replace_state(
            context,
            expected_phase="catalog-applied",
            updates={"phase": "governed-reference-data-in-progress"},
        )

    dispatch_env = dict(context.envv)
    dispatch_env["DEPLOY_ARTIFACT_DIR"] = str(
        context.run_root / "deploy-artifacts"
    )
    evidence = apply_seed(
        dispatch_env,
        l2_root,
        profile="test",
        phase="governed_reference_data",
    )
    selected_objects, reference_data_record_count = (
        validate_governed_reference_data_evidence(evidence)
    )

    evidence_bytes = canonical_bytes(evidence)
    evidence_path = context.run_root / "governed-reference-data-evidence.json"
    if evidence_path.is_symlink() or (
        evidence_path.exists() and not evidence_path.is_file()
    ):
        raise DeployError("governed reference-data evidence path is invalid")
    pending = evidence_path.with_suffix(".json.pending")
    pending.write_bytes(evidence_bytes)
    os.chmod(pending, 0o600)
    os.replace(pending, evidence_path)
    os.chmod(evidence_path, 0o400)
    receipt = {
        "schema": "nexuscrm.serviceification-d3-governed-reference-data/v1",
        "l2Commit": l2_commit,
        "sourceFingerprintSha256": evidence.get("source_fingerprint_sha256"),
        "objects": selected_objects,
        "recordCount": reference_data_record_count,
        "evidenceFile": str(evidence_path),
        "evidenceDigest": sha256_bytes(evidence_bytes),
    }
    return replace_state(
        context,
        expected_phase="governed-reference-data-in-progress",
        updates={
            "phase": "governed-reference-data-applied",
            "governedReferenceData": receipt,
        },
    )


def ensure_test2_inspection_identity(
    context: D3Context,
    core_root: Path,
    *,
    align: Callable[[dict[str, str], Path], None] | None = None,
) -> dict[str, Any]:
    """Align the test-only admin credential once for one adopted database baseline."""
    state = require_state(context)
    baseline = state.get("baselineReset")
    if not isinstance(baseline, dict) or state.get("testIdentityAligned") is not None:
        return state
    if align is None:
        align = align_runtime_admin_password
    align(dict(context.envv), core_root)
    return replace_state(
        context,
        expected_phase=str(state["phase"]),
        updates={
            "testIdentityAligned": {
                "schema": "nexuscrm.serviceification-d3-test2-inspection-identity/v1",
                "baselineSource": str(baseline.get("source", "")),
                "username": context.envv.get("VERIFY_USERNAME", "").strip()
                or "admin@example.invalid",
            },
        },
    )


def prepare_serving_inputs(
    context: D3Context,
    *,
    allow_loopback_http: bool = False,
    execute: Callable[[Sequence[str]], None] = default_execute,
    signing_key_provider: Callable[..., Path] | None = None,
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase="governed-reference-data-applied",
    )
    authority = verify_catalog_after(context, state)
    worker_ownership = require_worker_ownership_before(state)
    catalog_artifacts = state.get("catalogArtifacts")
    if not isinstance(catalog_artifacts, dict):
        raise DeployError("catalog-applied state has no sealed Catalog artifacts")
    catalog_output = Path(str(catalog_artifacts.get("output", "")))
    frontend_receipt = Path(str(catalog_artifacts.get("frontendReceipt", "")))
    catalog_store = Path(str(catalog_artifacts.get("store", "")))
    for path, label in (
        (catalog_output, "Catalog output"),
        (frontend_receipt, "frontend receipt"),
    ):
        if not path.is_file() or path.is_symlink():
            raise DeployError(f"sealed {label} disappeared before serving preparation")
    if not catalog_store.is_dir() or catalog_store.is_symlink():
        raise DeployError("Catalog snapshot store disappeared before serving preparation")

    private_root = context.run_root / "private"
    runtime_root = context.run_root / "runtime"
    worker_root = context.run_root / "workers"
    sfcrm_root = context.run_root / "sfcrm"
    integration_root = context.run_root / "integration"
    for path in (
        private_root / "runtime", private_root / "workloads",
        runtime_root, worker_root, sfcrm_root, integration_root,
    ):
        if path.is_symlink():
            raise DeployError(f"incomplete D3 serving path became a symlink: {path}")
        if path.exists():
            shutil.rmtree(path)
    for path in (
        private_root / "runtime", runtime_root / "config", runtime_root / "tls",
        worker_root / "activation-config", worker_root / "activation-output",
        worker_root / "config", worker_root / "secrets", sfcrm_root / "trust",
        integration_root / "trust",
    ):
        path.mkdir(parents=True, mode=0o700, exist_ok=False)

    tokens = provision_workload_identities(private_root / "workloads", mcp="mcp_worker" in context.images)
    if signing_key_provider is None:
        signing_key_provider = provision_runtime_signing_key
    signing_keys = signing_key_provider(private_root / "runtime", execute=execute)
    certificate, private_key = provision_tls_identity(
        runtime_root / "tls", RUNTIME_SERVER_NAME, execute=execute,
    )
    for trust_path in (
        sfcrm_root / "trust" / "runtime-ca.crt",
        integration_root / "trust" / "runtime-ca.crt",
    ):
        shutil.copyfile(certificate, trust_path)
        os.chmod(trust_path, 0o400)

    provider_document = d3_federated_oidc_document(context.envv)
    provider_path = runtime_root / "config" / "federated-oidc.json"
    if provider_document is not None:
        write_protected_json(provider_path, provider_document)
    runtime_config = runtime_root / "config" / "serving.json"
    render_serving = [
        "python3", str(REPOSITORY_ROOT / "scripts" / "render_serviceification_test_stack.py"),
        "serving", "--bom", str(context.run_root / "resolved-bom.json"),
        "--catalog-output", str(catalog_output),
        "--frontend-receipt", str(frontend_receipt),
        "--runtime-output", str(runtime_config),
        "--application-owner-id", str(state["runtimeAppShell"]["frontendAppId"]),
        "--replica-id", f"d3-test2-{context.run_id}",
        "--org-id", authority.org_id,
        "--workload-id", context.bom["catalog_consumers"]["handler_workload_id"],
        "--catalog-store-root", "/var/lib/nexuscrm/catalog",
    ]
    if provider_document is not None:
        render_serving.extend(("--federated-oidc-providers", str(provider_path)))
    execute(tuple(render_serving))
    if not runtime_config.is_file() or runtime_config.is_symlink():
        raise DeployError("D3 serving renderer emitted no regular Runtime config")
    os.chmod(runtime_config, 0o400)

    activation_config = worker_root / "activation-config"
    family_by_stem = {
        role["work_family"].removeprefix("runtime.").removesuffix(".v1").replace(".", "-"):
            role["work_family"]
        for role in context.bom["runtime_worker_roles"]
    }
    if set(family_by_stem) != set(context.worker_stems):
        raise DeployError("resolved D3 Worker role stems do not match the serving closure")
    for stem in context.worker_stems:
        path = activation_config / f"{stem}.json"
        family = family_by_stem[stem]
        before = worker_ownership[family]
        expected_epoch = 0 if before is None else before.ownership_epoch
        requests = render_worker_activation_inputs(
            dict(context.bom),
            deployment_id_prefix=f"d3-test2-{context.run_id}",
            expected_ownership_epoch=expected_epoch,
        )
        write_protected_json(path, requests[stem])
        provision_token(worker_root / "secrets" / f"{stem}.token")

    db_values = database_env(context.envv)
    storage_values = runtime_storage_env(context.envv)
    if storage_values["FILE_STORAGE_PROVIDER"] == "filesystem":
        ensure_isolated_runtime_filesystem_storage(context, storage_values)
    public_origin = validate_public_origin(
        required_env_value(context.envv, "PUBLIC_BASE_URL"),
        allow_loopback_http=allow_loopback_http,
    )
    connected_mail_oauth = connected_mail_oauth_env(context.envv)
    connected_mail_definition: Path | None = None
    connected_mail_values: dict[str, str] = {}
    if connected_mail_oauth:
        connected_mail_definition = materialize_connected_mail_definition(
            state, integration_root / "connected-mail" / "provider.json",
        )
        connected_mail_values = {
            **connected_mail_oauth,
            CONNECTED_MAIL_DEFINITION_ENV: CONNECTED_MAIL_CONTAINER_PATH,
            CONNECTED_MAIL_CALLBACK_ENV: public_origin + CONNECTED_MAIL_PUBLIC_PATH + "/callback",
            CONNECTED_MAIL_RETURN_BASE_ENV: public_origin,
            CONNECTED_MAIL_MASTER_KEY_ENV: required_env_value(
                context.envv, "NEXUSCRM_NC_MASTER_KEY",
            ),
        }
    runtime_values = {
        **db_values, **storage_values,
        # Match the proven staging foreground database capacity. Runtime's
        # independent admission gate is selected by the environment adapter,
        # so this pool size does not itself admit more RPCs.
        "NEXUSCRM_DB_MAX_OPEN_CONNS": "100",
        "NEXUSCRM_DB_MAX_IDLE_CONNS": "100",
        "NEXUSCRM_RUNTIME_API_ADMIN_ADDR": f"127.0.0.1:{context.ports.runtime_admin}",
        "NEXUSCRM_RUNTIME_API_GRPC_ADDR": f"127.0.0.1:{context.ports.runtime_grpc}",
        "NEXUSCRM_RUNTIME_API_ADMISSION_LIMIT": str(RUNTIME_API_ADMISSION_LIMIT),
        # Diagnostics are opt-in per bounded experiment. Leaving pprof and the
        # synchronous Flow/Records/SQL stage ledger enabled during ordinary
        # inspection changes the workload being measured and can dominate
        # remote-DB business paths. The serving-only operator can temporarily
        # enable this exact Runtime image, collect one bounded profile, then
        # return to the default without touching Catalog or release content.
        "NEXUSCRM_RUNTIME_API_PROFILING_ENABLED": "false",
        "NEXUSCRM_RUNTIME_API_SQL_DIAGNOSTICS_ENABLED": "false",
        "NEXUSCRM_RUNTIME_WORKLOAD_IDENTITY_TOKEN_FILE": "/run/secrets/workload-identities.json",
        "NEXUSCRM_RUNTIME_API_SIGNING_KEY_FILE": "/run/secrets/signing-keys.json",
        "NEXUSCRM_RUNTIME_API_SERVING_CONFIG_FILE": "/etc/nexuscrm/serving.json",
        "NEXUSCRM_RUNTIME_API_TLS_CERTIFICATE_FILE": "/run/tls/server.crt",
        "NEXUSCRM_RUNTIME_API_TLS_PRIVATE_KEY_FILE": "/run/tls/server.key",
        # The legacy Flow gate remains armed, but neither pprof nor the new
        # request/SQL diagnostic gate is enabled during ordinary serving.
        "NEXUSCRM_RUNTIME_FLOW_PERF_TIMING_ENABLED": "true",
        "JWT_SECRET": required_env_value(context.envv, "JWT_SECRET"),
        "API_BASE_URL": public_origin,
        # Runtime owns the neutral OAuth lifecycle. Public PKCE is a frozen D3
        # launch capability, so the candidate explicitly enables its kill switch
        # in the owning process rather than in the sfcrm HTTP projection.
        "NEXUSCRM_PUBLIC_OAUTH_PKCE_ENABLED": "true",
        **optional_env_values(context.envv, ("ENVIRONMENT", "NEXUS_ENV")),
    }
    write_env_file(private_root / "runtime.env", runtime_values)

    sfcrm_values = {
        "SFCRM_HTTP_ADDRESS": f"127.0.0.1:{context.ports.sfcrm_http}",
        "SFCRM_RUNTIME_TARGET": f"127.0.0.1:{context.ports.runtime_grpc}",
        "SFCRM_APP_INSTALLATION_ID": authority.app_installation_id,
        "SFCRM_PUBLIC_ORIGIN": public_origin,
        "SFCRM_ORGANIZATION_ID": authority.org_id,
        "SFCRM_SESSION_CONTROL_ORIGINS": public_origin,
        "SFCRM_WORKLOAD_TOKEN_FILE": "/run/secrets/workload.token",
        # Hosted Test2 uses bounded remote-database timing: the private attempt
        # ends before the Runtime lease, while the outer request owns the
        # complete Salesforce journey. This accommodates TiDB network variance
        # without changing the process admission cap.
        "SFCRM_REQUEST_TIMEOUT": "90s",
        "SFCRM_RUNTIME_ATTEMPT_TIMEOUT": D3_RUNTIME_ATTEMPT_BUDGET,
        "SFCRM_RUNTIME_STARTUP_TIMEOUT": "60s", "SFCRM_SHUTDOWN_TIMEOUT": "20s",
        "SFCRM_READ_HEADER_TIMEOUT": "10s", "SFCRM_IDLE_TIMEOUT": "60s",
        "SFCRM_GOOGLE_TIMEOUT": "15s", "SFCRM_MAX_REQUEST_BODY_BYTES": str(26 * 1024 * 1024),
        "SFCRM_RUNTIME_CA_FILE": "/run/trust/runtime-ca.crt",
        "SFCRM_RUNTIME_SERVER_NAME": RUNTIME_SERVER_NAME,
        **optional_env_values(context.envv, (
            "GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET", "NEXUS_LOGIN_METHODS",
            "NEXUS_PASSWORD_GRANT_ENABLED", "NEXUS_PASSWORD_GRANT_BREAKGLASS",
        )),
    }
    write_env_file(private_root / "sfcrm.env", sfcrm_values)
    if "mcp_worker" in context.images:
        write_env_file(private_root / "mcp.env", {
            "MCP_PUBLIC_ORIGIN": public_origin,
            "MCP_HTTP_ADDRESS": f"127.0.0.1:{context.ports.mcp_http}",
            "MCP_RUNTIME_TARGET": f"127.0.0.1:{context.ports.runtime_grpc}",
            "MCP_APP_INSTALLATION_ID": authority.app_installation_id,
            "MCP_WORKLOAD_TOKEN_FILE": "/run/secrets/workload.token",
            "MCP_RUNTIME_CA_FILE": "/run/trust/runtime-ca.crt",
            "MCP_RUNTIME_SERVER_NAME": RUNTIME_SERVER_NAME,
        })

    verified_ingress_receipts = required_env_value(
        context.envv, "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS",
    )
    if verified_ingress_receipts != "true":
        raise DeployError(
            "D3 Integration requires NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS=true"
        )
    integration_values = {
        **db_values,
        "NEXUSCRM_DB_MAX_OPEN_CONNS": "8",
        "NEXUSCRM_DB_MAX_IDLE_CONNS": "4",
        "NEXUSCRM_NC_MASTER_KEY": required_env_value(
            context.envv, "NEXUSCRM_NC_MASTER_KEY",
        ),
        "NEXUSCRM_INTEGRATION_HTTP_ADDR": f"127.0.0.1:{context.ports.integration_http}",
        "NEXUSCRM_INTEGRATION_ORG_ID": authority.org_id,
        "NEXUSCRM_INTEGRATION_APP_INSTALLATION_ID": authority.app_installation_id,
        "NEXUSCRM_INTEGRATION_AUTHORIZED_WORKLOAD_ID": "integration-service",
        "NEXUSCRM_INTEGRATION_WORKLOAD_TOKEN_FILE": "/run/secrets/workload.token",
        "NEXUSCRM_INTEGRATION_RUNTIME_TARGET": f"127.0.0.1:{context.ports.runtime_grpc}",
        "NEXUSCRM_INTEGRATION_RUNTIME_CA_FILE": "/run/trust/runtime-ca.crt",
        "NEXUSCRM_INTEGRATION_RUNTIME_SERVER_NAME": RUNTIME_SERVER_NAME,
        "NEXUSCRM_INTEGRATION_RUNTIME_ATTEMPT_TIMEOUT": "4s",
        "NEXUSCRM_INTEGRATION_RUNTIME_STARTUP_TIMEOUT": "60s",
        "NEXUSCRM_INTEGRATION_SHUTDOWN_TIMEOUT": "20s",
        "NEXUSCRM_INTEGRATION_READ_HEADER_TIMEOUT": "10s",
        "NEXUSCRM_INTEGRATION_REQUEST_TIMEOUT": "30s",
        "NEXUSCRM_INTEGRATION_IDLE_TIMEOUT": "60s",
        # Provider signatures bind the exact public request URL.  Once webhook
        # ownership moves out of the monolith, the Integration process must
        # receive both that public authority and its own durable-ingress
        # controls rather than silently falling back to connector defaults.
        "WEBHOOK_BASE_URL": public_origin,
        "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS": verified_ingress_receipts,
        **connected_mail_values,
        **optional_env_values(context.envv, (
            "NEXUSCRM_CONNECTOR_FLOW_DEADLINE_MS",
            "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS",
            "NEXUSCRM_CONNECTOR_INBOUND_QUEUE_PROFILE",
            "NEXUSCRM_CONNECTOR_INBOUND_STAGE_TIMING",
            "NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK",
            "NEXUSCRM_CONNECTOR_RECEIPT_POLL_INTERVAL_MS",
            "NEXUSCRM_CONNECTOR_RECEIPT_ROUTER_BATCH_SIZE",
            "NEXUSCRM_CONNECTOR_RECEIPT_ROUTER_PREPARE_CONCURRENCY",
            "NEXUSCRM_CONNECTOR_RECEIPT_ROUTER_PREPARE_TIMEOUT_MS",
            "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS",
            "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS",
            "NEXUSCRM_CONNECTOR_WEBHOOK_RETRY_AFTER_SECONDS",
            "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_LIMIT",
            "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_LIMIT",
            "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_MS",
        )),
    }
    write_env_file(private_root / "integration.env", integration_values)

    for stem in context.worker_stems:
        write_env_file(
            private_root / f"worker-{stem}.env",
            worker_environment_values(context, stem),
        )

    artifacts = {
        "runtimeConfig": str(runtime_config),
        "runtimeConfigDigest": sha256_bytes(runtime_config.read_bytes()),
        "runtimeEnv": str(private_root / "runtime.env"),
        "runtimeSigningKeys": str(signing_keys),
        "runtimeWorkloadIdentities": str(private_root / "workloads" / "runtime-workload-identities.json"),
        "runtimeCertificate": str(certificate), "runtimePrivateKey": str(private_key),
        **({
            "mcpEnv": str(private_root / "mcp.env"),
            "mcpToken": str(private_root / "workloads" / "mcp.token"),
        } if "mcp_worker" in context.images else {}),
        "sfcrmEnv": str(private_root / "sfcrm.env"),
        "sfcrmToken": str(private_root / "workloads" / "sfcrm.token"),
        "sfcrmTrust": str(sfcrm_root / "trust" / "runtime-ca.crt"),
        "integrationEnv": str(private_root / "integration.env"),
        "integrationToken": str(private_root / "workloads" / "integration.token"),
        "integrationTrust": str(integration_root / "trust" / "runtime-ca.crt"),
        **({
            "connectedMailDefinition": str(connected_mail_definition),
            "connectedMailDefinitionDigest": sha256_bytes(
                connected_mail_definition.read_bytes(),
            ),
        } if connected_mail_definition is not None else {}),
        "workerActivationConfig": str(activation_config),
        "workerActivationConfigDigests": {
            stem: sha256_bytes((activation_config / f"{stem}.json").read_bytes())
            for stem in context.worker_stems
        },
        "workerActivationOutput": str(worker_root / "activation-output"),
        "workerConfig": str(worker_root / "config"),
        "workerSecrets": str(worker_root / "secrets"),
        **({"federatedOIDC": str(provider_path)} if provider_document is not None else {}),
    }
    replace_state(
        context, expected_phase="governed-reference-data-applied",
        updates={"phase": "serving-inputs-ready", "servingArtifacts": artifacts},
    )
    return {
        "schema": EVIDENCE_SCHEMA, "stage": "serving-inputs-ready",
        "runId": context.run_id, "runtimeEndpointCount": 1,
        "workerRoleCount": len(context.worker_stems), "workloadIdentityCount": len(tokens),
        "federatedProviderCount": 1 if provider_document is not None else 0,
    }


def activate_worker_ownership(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={"serving-inputs-ready", "worker-activation-in-progress"},
    )
    authority = verify_catalog_after(context, state)
    before = require_worker_ownership_before(state)
    artifacts = state.get("servingArtifacts")
    catalog_artifacts = state.get("catalogArtifacts")
    if not isinstance(artifacts, dict) or not isinstance(catalog_artifacts, dict):
        raise DeployError("D3 serving or Catalog artifacts are incomplete")
    activation_dir = Path(str(artifacts.get("workerActivationConfig", "")))
    output_dir = Path(str(artifacts.get("workerActivationOutput", "")))
    role_dir = Path(str(artifacts.get("workerConfig", "")))
    catalog_output_path = Path(str(catalog_artifacts.get("output", "")))
    db_env_file = Path(str(catalog_artifacts.get("databaseEnv", "")))
    expected_digests = artifacts.get("workerActivationConfigDigests")
    if not isinstance(expected_digests, dict) or set(expected_digests) != set(context.worker_stems):
        raise DeployError("D3 Worker activation inputs have no complete digest seal")
    activation_requests: dict[str, dict[str, Any]] = {}
    for stem in context.worker_stems:
        path = activation_dir / f"{stem}.json"
        if sha256_bytes(path.read_bytes()) != expected_digests[stem]:
            raise DeployError(f"sealed Runtime Worker activation input changed: {stem}")
        activation_requests[stem] = load_json_object(
            path, f"Runtime Worker activation input {stem}",
        )
    catalog_output = load_json_object(catalog_output_path, "sealed Catalog output")
    if (
        not db_env_file.is_file() or db_env_file.is_symlink()
        or db_env_file.stat().st_mode & 0o077
    ):
        raise DeployError("Catalog database credential file is missing or disclosed")

    if state["phase"] == "serving-inputs-ready":
        if read_worker_ownership(context.envv) != before:
            raise DeployError("Runtime Worker ownership changed after D3 quiescence")
        replace_state(
            context,
            expected_phase="serving-inputs-ready",
            updates={"phase": "worker-activation-in-progress"},
        )

    activation_outputs: dict[str, dict[str, Any]] = {}
    for stem in context.worker_stems:
        output = output_dir / f"{stem}.json"
        if not output.exists() and not output.is_symlink():
            container = container_name(context.run_id, f"worker-activation-{stem}")
            execute((
                "docker", "run", "--rm", "--name", container,
                "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
                "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
                "--env-file", str(db_env_file),
                "--mount", (
                    f"type=bind,src={activation_dir / f'{stem}.json'},"
                    "dst=/etc/nexuscrm/work-ownership.json,readonly"
                ),
                "--mount", f"type=bind,src={output_dir},dst=/output",
                context.images["catalog_reconcile"],
                "work-ownership", "-config", "/etc/nexuscrm/work-ownership.json",
                "-output", f"/output/{stem}.json",
            ))
        activation_outputs[stem] = load_json_object(
            output, f"Runtime Worker activation output {stem}",
        )
        os.chmod(output, 0o400)

    try:
        role_configs = render_worker_role_inputs(
            dict(context.bom), catalog_output,
            activation_requests, activation_outputs,
            org_id=authority.org_id,
            workload_id=context.bom["catalog_consumers"]["handler_workload_id"],
            catalog_store_root="/var/lib/nexuscrm/catalog",
        )
    except ValidationFailure as exc:
        raise DeployError(f"Runtime Worker ownership receipts are not deployable: {exc}") from exc
    for stem in context.worker_stems:
        path = role_dir / f"{stem}.json"
        expected = canonical_bytes(role_configs[stem])
        if path.exists() or path.is_symlink():
            if not path.is_file() or path.is_symlink() or path.read_bytes() != expected:
                raise DeployError(f"Runtime Worker role config changed during recovery: {stem}")
        else:
            write_protected_json(path, role_configs[stem])

    current = read_worker_ownership(context.envv)
    expected_after: dict[str, WorkerOwnership | None] = {
        family: None for family in SUPPORTED_WORKER_FAMILIES
    }
    for stem, output in activation_outputs.items():
        family = str(output.get("workFamily", ""))
        try:
            epoch = int(str(output.get("ownershipEpoch", "")))
        except ValueError as exc:
            raise DeployError(f"Runtime Worker activation output has invalid epoch: {stem}") from exc
        expected_after[family] = WorkerOwnership(
            work_family=family,
            deployment_id=str(output.get("deploymentId", "")),
            worker_release_digest=str(output.get("workerReleaseDigest", "")),
            ownership_epoch=epoch,
            active=True,
            operation_id=str(output.get("operationId", "")),
        )
    if current != expected_after:
        raise DeployError("Runtime Worker ownership authority does not match every selected receipt")

    activation_output_digests = {
        stem: sha256_bytes((output_dir / f"{stem}.json").read_bytes())
        for stem in context.worker_stems
    }
    role_config_digests = {
        stem: sha256_bytes((role_dir / f"{stem}.json").read_bytes())
        for stem in context.worker_stems
    }
    next_state = replace_state(
        context,
        expected_phase="worker-activation-in-progress",
        updates={
            "phase": "worker-owned",
            "workerOwnershipAfter": worker_ownership_payload(expected_after),
            "workerActivationOutputDigests": activation_output_digests,
            "workerRoleConfigDigests": role_config_digests,
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "worker-owned",
        "runId": context.run_id,
        "workFamilyCount": len(context.worker_stems),
        "workerOwnershipAfter": next_state["workerOwnershipAfter"],
    }


def wait_http_ready(
    url: str,
    container: str,
    *,
    timeout_seconds: float = D3_SERVING_READINESS_TIMEOUT_SECONDS,
    probe: Callable[[Sequence[str]], subprocess.CompletedProcess[str]] | None = None,
    container_running: Callable[[str], bool] | None = None,
    container_restart_count: Callable[[str], int | None] | None = None,
) -> None:
    if probe is None:
        def probe(args: Sequence[str]) -> subprocess.CompletedProcess[str]:
            return subprocess.run(
                list(args), check=False, capture_output=True, text=True,
            )
    if container_running is None:
        def container_running(name: str) -> bool:
            result = subprocess.run(
                ("docker", "inspect", "--format", "{{.State.Running}}", name),
                check=False, capture_output=True, text=True,
            )
            return result.returncode == 0 and result.stdout.strip() == "true"
    if container_restart_count is None:
        def container_restart_count(name: str) -> int | None:
            result = subprocess.run(
                ("docker", "inspect", "--format", "{{.RestartCount}}", name),
                check=False, capture_output=True, text=True,
            )
            if result.returncode != 0:
                return None
            try:
                return int(result.stdout.strip())
            except ValueError:
                return None
    deadline = time.monotonic() + timeout_seconds
    while time.monotonic() < deadline:
        result = probe((
            "curl", "--fail", "--silent", "--show-error",
            "--max-time", "2", url,
        ))
        if result.returncode == 0:
            return
        if not container_running(container):
            raise DeployError(f"D3 container exited before readiness: {container} ({url})")
        restart_count = container_restart_count(container)
        if restart_count is not None and restart_count > 0:
            raise DeployError(
                f"D3 container restarted before readiness: {container} "
                f"(restart_count={restart_count}, {url})"
            )
        time.sleep(1)
    raise DeployError(f"D3 container did not become ready: {container} ({url})")


def wait_http_ready_after_existing_restart(
    url: str,
    container: str,
    *,
    timeout_seconds: float = D3_SERVING_READINESS_TIMEOUT_SECONDS,
) -> None:
    """Wait for a recovered container without forgiving another restart.

    A failed replacement can leave the restored previous pair running after
    one startup restart.  Recovery may accept that historical restart only
    after the exact previous pair is proven; any additional restart while this
    probe is active still fails closed.
    """
    baseline = default_capture((
        "docker", "inspect", "--format", "{{.RestartCount}}", container,
    ))
    try:
        baseline_restarts = int(baseline)
    except ValueError as exc:
        raise DeployError(
            f"D3 recovery cannot read container restart count: {container}",
        ) from exc
    deadline = time.monotonic() + timeout_seconds
    while time.monotonic() < deadline:
        result = subprocess.run(
            (
                "curl", "--fail", "--silent", "--show-error",
                "--max-time", "2", url,
            ),
            check=False,
            capture_output=True,
            text=True,
        )
        if result.returncode == 0:
            return
        running = default_capture((
            "docker", "inspect", "--format", "{{.State.Running}}", container,
        ))
        if running != "true":
            raise DeployError(
                f"D3 recovered container exited before readiness: {container} ({url})",
            )
        current = default_capture((
            "docker", "inspect", "--format", "{{.RestartCount}}", container,
        ))
        try:
            current_restarts = int(current)
        except ValueError as exc:
            raise DeployError(
                f"D3 recovery cannot read container restart count: {container}",
            ) from exc
        if current_restarts != baseline_restarts:
            raise DeployError(
                f"D3 recovered container restarted again before readiness: {container} "
                f"(baseline={baseline_restarts}, current={current_restarts}, {url})",
            )
        time.sleep(1)
    raise DeployError(f"D3 recovered container did not become ready: {container} ({url})")


def mount_argument(source: Path, destination: str, *, readonly: bool = True) -> str:
    if source.is_symlink():
        raise DeployError(f"D3 container mount source is unavailable: {source}")
    try:
        source = source.resolve(strict=True)
    except FileNotFoundError as exc:
        raise DeployError(f"D3 container mount source is unavailable: {source}") from exc
    if not destination.startswith("/") or "," in str(source) or "," in destination:
        raise DeployError("D3 container mount path is not canonical")
    value = f"type=bind,src={source},dst={destination}"
    return value + (",readonly" if readonly else "")


def verify_runtime_filesystem_storage_write_path(
    context: D3Context,
    *,
    root: Path,
    runtime_image: str,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> None:
    """Prove a real nested atomic upload path before Runtime starts."""
    probe_directory = f".nexuscrm-runtime-write-probe-{context.run_id}"
    script = f"""
set -eu
probe_root="/storage/system-file/{probe_directory}"
attempt_root="${{probe_root}}/.attempts"
first="${{attempt_root}}/object.tmp"
second="${{attempt_root}}/object.committed"
mkdir -p "${{attempt_root}}"
umask 077
printf x > "${{first}}"
mv "${{first}}" "${{second}}"
rm "${{second}}"
rmdir "${{attempt_root}}"
rmdir "${{probe_root}}"
rmdir /storage/system-file 2>/dev/null || true
""".strip()
    execute((
        "docker", "run", "--rm", "--pull", "never",
        "--name", container_name(context.run_id, "storage-write-probe"),
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        "--network", "none", "--user", f"{os.getuid()}:{os.getgid()}",
        "--mount", mount_argument(root, "/storage", readonly=False),
        "--entrypoint", "/bin/sh", runtime_image, "-ceu", script,
    ))


def verify_runtime_storage_environment_paths(
    storage: Mapping[str, str],
    environment_paths: Sequence[Path],
) -> None:
    """Reject a serving environment that names a different storage authority."""
    for environment_path in environment_paths:
        serving_storage = load_env_file(environment_path)
        if serving_storage.get("FILE_STORAGE_PROVIDER", "filesystem") != storage[
            "FILE_STORAGE_PROVIDER"
        ]:
            raise DeployError("serving filesystem storage provider changed authority")
        if (
            storage["FILE_STORAGE_PROVIDER"] == "filesystem"
            and serving_storage.get("FILE_STORAGE_ROOT") != storage["FILE_STORAGE_ROOT"]
        ):
            raise DeployError(
                "serving filesystem storage root predates the isolated authority"
            )


def preflight_runtime_filesystem_storage_mounts(
    context: D3Context,
    *,
    runtime_image: str,
    environment_paths: Sequence[Path] = (),
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> tuple[tuple[Path, str, bool], ...]:
    """Return the canonical mounts only after their host and container proof."""
    storage = runtime_storage_env(context.envv)
    verify_runtime_storage_environment_paths(storage, environment_paths)
    if storage["FILE_STORAGE_PROVIDER"] != "filesystem":
        return ()
    storage_root = ensure_isolated_runtime_filesystem_storage(context, storage)
    verify_runtime_filesystem_storage_write_path(
        context,
        root=storage_root,
        runtime_image=runtime_image,
        execute=execute,
    )
    return ((storage_root, storage["FILE_STORAGE_ROOT"], False),)


def docker_healthcheck_command(ready_url: str) -> str:
    try:
        parsed = urllib.parse.urlsplit(ready_url)
        port = parsed.port
    except ValueError as exc:
        raise DeployError(
            "D3 healthcheck URL must use one canonical loopback ready endpoint",
        ) from exc
    if (
        parsed.scheme != "http"
        or parsed.hostname != "127.0.0.1"
        or parsed.username is not None
        or parsed.password is not None
        or port is None
        or not 1 <= port <= 65535
        or parsed.netloc != f"127.0.0.1:{port}"
        or parsed.path != "/readyz"
        or parsed.query
        or parsed.fragment
    ):
        raise DeployError(
            "D3 healthcheck URL must use one canonical loopback ready endpoint",
        )
    return f"wget -qO- {ready_url} >/dev/null || exit 1"


def wait_docker_healthy(
    ready_url: str,
    container: str,
    *,
    timeout_seconds: float = D3_DOCKER_HEALTH_TIMEOUT_SECONDS,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> None:
    expected_test = ["CMD-SHELL", docker_healthcheck_command(ready_url)]
    raw_test = capture_output((
        "docker", "inspect", "--format", "{{json .Config.Healthcheck.Test}}",
        container,
    ))
    try:
        actual_test = json.loads(raw_test)
    except json.JSONDecodeError as exc:
        raise DeployError(
            f"D3 container healthcheck is unreadable: {container}",
        ) from exc
    if actual_test != expected_test:
        raise DeployError(
            f"D3 container healthcheck drifted: {container} "
            f"(expected={ready_url})",
        )

    deadline = time.monotonic() + timeout_seconds
    last_status = "unknown"
    while time.monotonic() < deadline:
        fields = capture_output((
            "docker", "inspect", "--format",
            '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}\t'
            "{{.State.Running}}",
            container,
        )).split("\t")
        if len(fields) != 2 or fields[0] not in {"starting", "healthy", "unhealthy"}:
            raise DeployError(f"D3 container health state is unknown: {container}")
        last_status = fields[0]
        if fields[1] != "true":
            raise DeployError(
                f"D3 container exited before Docker health converged: {container}",
            )
        if last_status == "healthy":
            return
        time.sleep(1)
    raise DeployError(
        f"D3 container did not become Docker healthy: {container} "
        f"(status={last_status}, expected={ready_url})",
    )


def run_detached_container(
    context: D3Context,
    *,
    name: str,
    image: str,
    env_file: Path,
    mounts: Sequence[tuple[Path, str, bool]],
    healthcheck_url: str | None,
    execute: Callable[[Sequence[str]], None],
) -> None:
    command: list[str] = [
        "docker", "run", "--pull", "never", "--detach", "--name", name,
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        "--label", f"io.nexuscrm.serviceification-d3.run={context.run_id}",
        "--network", "host", "--restart", "unless-stopped",
        "--user", f"{os.getuid()}:{os.getgid()}",
        "--env-file", str(env_file.resolve()),
    ]
    if not env_file.is_file() or env_file.is_symlink() or env_file.stat().st_mode & 0o077:
        raise DeployError(f"D3 container environment is missing or disclosed: {env_file}")
    for source, destination, readonly in mounts:
        command.extend(("--mount", mount_argument(source, destination, readonly=readonly)))
    if healthcheck_url is not None:
        command.extend((
            "--health-cmd", docker_healthcheck_command(healthcheck_url),
            "--health-interval", "5s",
            "--health-timeout", "3s",
            "--health-retries", "12",
            "--health-start-period", "5s",
        ))
    command.append(image)
    execute(tuple(command))


def ensure_detached_container(
    context: D3Context,
    *,
    name: str,
    image: str,
    env_file: Path,
    mounts: Sequence[tuple[Path, str, bool]],
    ready_url: str,
    healthcheck_url: str | None,
    present: set[str],
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
    wait_ready: Callable[[str, str], None],
    wait_healthy: Callable[[str, str], None] = wait_docker_healthy,
) -> None:
    if name in present:
        running = verify_container_identity(
            context, name, image, capture_output=capture_output,
        )
        if not running:
            execute(("docker", "start", name))
    else:
        run_detached_container(
            context, name=name, image=image, env_file=env_file,
            mounts=mounts, healthcheck_url=healthcheck_url, execute=execute,
        )
        present.add(name)
    wait_ready(ready_url, name)
    if healthcheck_url is not None:
        wait_healthy(healthcheck_url, name)


def require_worker_ownership_after(
    state: Mapping[str, Any],
) -> dict[str, WorkerOwnership | None]:
    renamed = {**dict(state), "workerOwnershipBefore": state.get("workerOwnershipAfter")}
    return require_worker_ownership_before(renamed)


def worker_role_config_path(state: Mapping[str, Any], stem: str) -> Path:
    """Resolve one sealed L0 role config, including an active runtime override."""
    stems = state_worker_stems(state)
    if stem not in stems:
        raise DeployError(f"unknown Runtime Worker role: {stem}")
    artifacts = state.get("servingArtifacts")
    if not isinstance(artifacts, dict):
        raise DeployError("D3 serving state has no Worker configuration owner")
    overrides = state.get("workerRoleConfigOverrides", {})
    if not isinstance(overrides, dict) or not set(overrides).issubset(stems):
        raise DeployError("D3 serving state has an invalid Worker runtime override map")
    raw = overrides.get(stem)
    path = (
        Path(str(raw)) if isinstance(raw, str) and raw
        else Path(str(artifacts.get("workerConfig", ""))) / f"{stem}.json"
    )
    if not path.is_file() or path.is_symlink():
        raise DeployError(f"sealed Runtime Worker role config is unavailable: {stem}")
    return path


def effective_runtime_worker_role_policies(
    state: Mapping[str, Any], active_bom: Mapping[str, Any],
) -> list[dict[str, Any]]:
    """Project sealed runtime overrides onto the immutable release role policies."""
    verify_worker_role_config_seals(state)
    roles = active_bom.get("runtime_worker_roles")
    if not isinstance(roles, list):
        raise DeployError("active BOM has no Runtime Worker role policies")
    effective: list[dict[str, Any]] = []
    for role in roles:
        if not isinstance(role, dict):
            raise DeployError("active BOM has an invalid Runtime Worker role policy")
        family = role.get("work_family")
        schema = role.get("config_schema")
        if not isinstance(family, str) or not family.startswith("runtime.") or not family.endswith(".v1"):
            raise DeployError("active BOM has an invalid Runtime Worker family")
        stem = worker_stem(family)
        if stem not in state_worker_stems(state):
            raise DeployError(f"active BOM has an unknown Runtime Worker family: {family}")
        config = load_json_object(
            worker_role_config_path(state, stem),
            f"effective Runtime Worker role config {stem}",
        )
        if config.get("work_family") != family or config.get("schema") != schema:
            raise DeployError(f"effective Runtime Worker role authority changed identity: {stem}")
        projected = copy.deepcopy(role)
        for key in role:
            if key in {"work_family", "config_schema"}:
                continue
            if key not in config:
                raise DeployError(f"effective Runtime Worker role config lost {key}: {stem}")
            projected[key] = copy.deepcopy(config[key])
        effective.append(projected)
    return effective


def verify_worker_role_config_seals(state: Mapping[str, Any]) -> dict[str, str]:
    stems = state_worker_stems(state)
    digests = state.get("workerRoleConfigDigests")
    if not isinstance(digests, dict) or set(digests) != set(stems):
        raise DeployError("D3 serving state has no complete Worker role digest seal")
    for stem in stems:
        if sha256_bytes(worker_role_config_path(state, stem).read_bytes()) != digests[stem]:
            raise DeployError(f"sealed Runtime Worker role config changed: {stem}")
    return {stem: str(digests[stem]) for stem in stems}


def worker_container_mounts(
    context: D3Context, state: Mapping[str, Any], stem: str, role_path: Path,
) -> tuple[tuple[Path, str, bool], ...]:
    artifacts = state.get("servingArtifacts")
    catalog_artifacts = state.get("catalogArtifacts")
    if not isinstance(artifacts, dict) or not isinstance(catalog_artifacts, dict):
        raise DeployError("D3 serving state has incomplete Worker serving artifacts")
    mounts: list[tuple[Path, str, bool]] = [
        (role_path, "/etc/nexuscrm/worker-role.json", True),
        (
            Path(str(artifacts["workerSecrets"])) / f"{stem}.token",
            "/run/secrets/workload.token", True,
        ),
        (Path(str(catalog_artifacts["store"])), "/var/lib/nexuscrm/catalog", True),
    ]
    if stem == "record-event-dispatch":
        mounts.append((
            worker_role_config_path(state, "scheduled-flow"),
            "/etc/nexuscrm/scheduled-flow-owner.json", True,
        ))
    if stem in OBJECT_STORAGE_WORKER_STEMS:
        storage = runtime_storage_env(context.envv)
        if storage["FILE_STORAGE_PROVIDER"] == "filesystem":
            mounts.append((
                Path(storage["FILE_STORAGE_ROOT"]), storage["FILE_STORAGE_ROOT"], False,
            ))
    return tuple(mounts)


def reestablish_prod_clone_serving_quiescence(
    context: D3Context,
    state: Mapping[str, Any],
    *,
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
) -> None:
    baseline = state.get("baselineReset")
    if not isinstance(baseline, dict) or baseline.get("source") not in RESET_SNAPSHOT_BASELINE_SOURCES:
        return

    # A failed serving attempt may leave earlier roles alive while a later role
    # is rejected. Stop that partial tuple before proving the old writer is
    # quiescent again; otherwise the retry itself can race durable work creation.
    expected_images = container_images_by_name(context, state)
    present = set(owned_containers())
    if not present.issubset(expected_images):
        raise DeployError("D3 serving retry found a container outside the exact run inventory")
    for name in reversed(expected_serving_container_order(state)):
        if name not in present:
            continue
        if verify_container_identity(
            context, name, expected_images[name], capture_output=capture_output,
        ):
            execute(("docker", "stop", "--time", "30", name))

    staging.quiesce_staging_target_runtime(
        dict(context.envv), runtime_paths(context.env_root)["run_dir"],
        disable_autostart=True,
    )
    units = subprocess.run(
        (
            "systemctl", "list-units", "--type=service", "--state=running",
            "--no-legend", "--plain", "nexuscrm*.service",
        ),
        check=False, capture_output=True, text=True,
    )
    if units.returncode != 0:
        raise DeployError(
            "cannot prove zero legacy NexusCRM systemd processes: "
            + (units.stderr.strip() or f"systemctl exit {units.returncode}")
        )
    active_units = sorted({
        line.split()[0] for line in units.stdout.splitlines() if line.split()
    })
    if active_units:
        raise DeployError(
            "legacy NexusCRM systemd processes remain active after target quiescence: "
            + ", ".join(active_units)
        )
    proc_root = Path("/proc")
    legacy_processes: set[str] = set()
    if proc_root.is_dir():
        for process in proc_root.iterdir():
            if not process.name.isdigit():
                continue
            try:
                executable = (process / "exe").resolve(strict=True).name
            except (FileNotFoundError, PermissionError, OSError):
                continue
            if executable.startswith("backend-server-"):
                legacy_processes.add(executable)
    if legacy_processes:
        raise DeployError(
            "legacy NexusCRM backend processes remain active after target quiescence: "
            + ", ".join(sorted(legacy_processes))
        )
    docker_processes = subprocess.run(
        ("docker", "ps", "--format", "{{.Names}}\t{{.Image}}"),
        check=False, capture_output=True, text=True,
    )
    if docker_processes.returncode != 0:
        raise DeployError(
            "cannot prove zero legacy NexusCRM Docker processes: "
            + (docker_processes.stderr.strip() or f"docker ps exit {docker_processes.returncode}")
        )
    legacy_containers: list[str] = []
    for line in docker_processes.stdout.splitlines():
        name, separator, image = line.partition("\t")
        if separator and "nexuscrm" in f"{name}\t{image}".lower():
            legacy_containers.append(f"{name} ({image})")
    if legacy_containers:
        raise DeployError(
            "legacy NexusCRM Docker processes remain active after D3 tuple stop: "
            + ", ".join(sorted(legacy_containers))
        )


def start_serving(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={"worker-owned", "serving-start-in-progress"},
    )
    reestablish_prod_clone_serving_quiescence(
        context, state, execute=execute, capture_output=capture_output,
    )
    baseline = state.get("baselineReset")
    verify_catalog_after(context, state)
    if read_worker_ownership(context.envv) != require_worker_ownership_after(state):
        raise DeployError("Runtime Worker ownership changed before D3 serving start")
    if context.serving_environment_preparer is not None:
        state = context.serving_environment_preparer(
            context,
            state,
            execute=execute,
            capture_output=capture_output,
        )
    artifacts = state.get("servingArtifacts")
    catalog_artifacts = state.get("catalogArtifacts")
    containers = state.get("containers")
    if (
        not isinstance(artifacts, dict)
        or not isinstance(catalog_artifacts, dict)
        or not isinstance(containers, dict)
        or not isinstance(containers.get("workers"), dict)
    ):
        raise DeployError("D3 serving state has incomplete artifact or container ownership")
    verify_worker_role_config_seals(state)

    catalog_store = Path(str(catalog_artifacts["store"]))
    service_images = effective_service_images(context, state)
    storage_mounts = preflight_runtime_filesystem_storage_mounts(
        context,
        runtime_image=service_images["runtime_api"],
        environment_paths=(
            Path(str(artifacts["runtimeEnv"])),
            *(
                context.run_root / "private" / f"worker-{stem}.env"
                for stem in context.worker_stems
                if stem in OBJECT_STORAGE_WORKER_STEMS
            ),
        ),
        execute=execute,
    )

    expected_order = expected_serving_container_order(state)
    expected_images = container_images_by_name(context, state)
    present = set(owned_containers())
    if not present.issubset(expected_images):
        raise DeployError("D3 serving recovery found a container outside the exact run inventory")
    if state["phase"] == "worker-owned":
        replace_state(
            context,
            expected_phase="worker-owned",
            updates={"phase": "serving-start-in-progress"},
        )
    started: list[str] = []
    runtime_name = str(containers["runtime_api"])
    ensure_detached_container(
        context, name=runtime_name, image=service_images["runtime_api"],
        env_file=Path(str(artifacts["runtimeEnv"])),
        mounts=(
            (Path(str(artifacts["runtimeConfig"])), "/etc/nexuscrm/serving.json", True),
            (Path(str(artifacts["runtimeWorkloadIdentities"])), "/run/secrets/workload-identities.json", True),
            (Path(str(artifacts["runtimeSigningKeys"])), "/run/secrets/signing-keys.json", True),
            (Path(str(artifacts["runtimeCertificate"])), "/run/tls/server.crt", True),
            (Path(str(artifacts["runtimePrivateKey"])), "/run/tls/server.key", True),
            (catalog_store, "/var/lib/nexuscrm/catalog", True),
            *storage_mounts,
        ), ready_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
        healthcheck_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
        present=present, execute=execute, capture_output=capture_output,
        wait_ready=wait_ready,
    )
    started.append(runtime_name)
    worker_names: dict[str, str] = {}
    worker_image = effective_worker_image(context, state)
    for stem in context.worker_stems:
        name = str(containers["workers"][stem])
        role_path = worker_role_config_path(state, stem)
        ensure_detached_container(
            context, name=name, image=worker_image,
            env_file=context.run_root / "private" / f"worker-{stem}.env",
            mounts=worker_container_mounts(context, state, stem, role_path),
            ready_url=f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz",
            healthcheck_url=f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz",
            present=present, execute=execute, capture_output=capture_output,
            wait_ready=wait_ready,
        )
        started.append(name)
        worker_names[stem] = name

    integration_name = str(containers["integration_service"])
    integration_mounts = [
        (Path(str(artifacts["integrationToken"])), "/run/secrets/workload.token", True),
        (Path(str(artifacts["integrationTrust"])), "/run/trust/runtime-ca.crt", True),
    ]
    if "connectedMailDefinition" in artifacts:
        integration_mounts.append((
            Path(str(artifacts["connectedMailDefinition"])),
            CONNECTED_MAIL_CONTAINER_PATH,
            True,
        ))
    ensure_detached_container(
        context, name=integration_name, image=service_images["integration_service"],
        env_file=Path(str(artifacts["integrationEnv"])),
        mounts=tuple(integration_mounts),
        ready_url=f"http://127.0.0.1:{context.ports.integration_http}/readyz",
        healthcheck_url=None,
        present=present, execute=execute, capture_output=capture_output,
        wait_ready=wait_ready,
    )
    started.append(integration_name)

    sfcrm_name = str(containers["sfcrm_app_service"])
    ensure_detached_container(
        context, name=sfcrm_name, image=service_images["sfcrm_app_service"],
        env_file=Path(str(artifacts["sfcrmEnv"])),
        mounts=(
            (Path(str(artifacts["sfcrmToken"])), "/run/secrets/workload.token", True),
            (Path(str(artifacts["sfcrmTrust"])), "/run/trust/runtime-ca.crt", True),
        ), ready_url=f"http://127.0.0.1:{context.ports.sfcrm_http}/readyz",
        healthcheck_url=None,
        present=present, execute=execute, capture_output=capture_output,
        wait_ready=wait_ready,
    )
    started.append(sfcrm_name)
    if "mcp_worker" in context.images:
        start_mcp_worker(context, state, present=present, execute=execute,
                         capture_output=capture_output, wait_ready=wait_ready)
        started.append(str(containers["mcp_worker"]))

    ingress_state = ensure_d3_public_ingress(context)
    frontend = Path(str(ingress_state["publishedFrontendTarget"])).resolve()

    next_state = replace_state(
        context,
        expected_phase="serving-start-in-progress",
        updates={
            "phase": "serving",
            "startedContainers": started,
            "publishedFrontendTarget": str(frontend),
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "serving",
        "runId": context.run_id,
        "runtimeEndpointCount": 1,
        "workerRoleCount": len(worker_names),
        "containers": next_state["startedContainers"],
        "frontendTarget": next_state["publishedFrontendTarget"],
    }


def start_mcp_worker(context: D3Context, state: Mapping[str, Any], *, present: set[str],
                     execute=default_execute, capture_output=default_capture, wait_ready=wait_http_ready) -> None:
    artifacts = state["servingArtifacts"]
    ensure_detached_container(
        context, name=str(state["containers"]["mcp_worker"]), image=context.images["mcp_worker"],
        env_file=Path(str(artifacts["mcpEnv"])),
        mounts=((Path(str(artifacts["mcpToken"])), "/run/secrets/workload.token", True),
                (Path(str(artifacts["sfcrmTrust"])), "/run/trust/runtime-ca.crt", True)),
        ready_url=f"http://127.0.0.1:{context.ports.mcp_http}/readyz", healthcheck_url=None,
        present=present, execute=execute, capture_output=capture_output, wait_ready=wait_ready,
    )


def expected_serving_container_order(state: Mapping[str, Any]) -> list[str]:
    stems = state_worker_stems(state)
    containers = state.get("containers")
    if not isinstance(containers, dict) or not isinstance(containers.get("workers"), dict):
        raise DeployError("D3 state has no exact serving container inventory")
    workers = containers["workers"]
    if set(workers) != set(stems):
        raise DeployError("D3 state has an incomplete Worker container inventory")
    return [
        str(containers["runtime_api"]),
        *(str(workers[stem]) for stem in stems),
        str(containers["integration_service"]),
        str(containers["sfcrm_app_service"]),
        *([str(containers["mcp_worker"])] if "mcp_worker" in containers else []),
    ]


def verify_container_identity(
    context: D3Context,
    name: str,
    expected_image: str,
    *,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> bool:
    value = capture_output((
        "docker", "inspect", "--format",
        '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}\t'
        '{{.HostConfig.NetworkMode}}\t{{.HostConfig.RestartPolicy.Name}}\t{{.State.Running}}',
        name,
    ))
    fields = value.split("\t")
    if len(fields) != 5 or fields[:4] != [
        expected_image, context.run_id, "host", "unless-stopped",
    ]:
        raise DeployError(f"D3 container identity drifted: {name}")
    if fields[4] not in {"true", "false"}:
        raise DeployError(f"D3 container state is unknown: {name}")
    return fields[4] == "true"


def converge_container_supervision(
    context: D3Context,
    name: str,
    expected_image: str,
    *,
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
) -> None:
    """Adopt the final restart policy without accepting another container identity."""
    inspect = (
        "docker", "inspect", "--format",
        '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}\t'
        '{{.HostConfig.NetworkMode}}\t{{.HostConfig.RestartPolicy.Name}}\t{{.State.Running}}',
        name,
    )
    fields = capture_output(inspect).split("\t")
    if (
        len(fields) != 5
        or fields[:3] != [expected_image, context.run_id, "host"]
        or fields[3] not in {"no", "unless-stopped"}
        or fields[4] not in {"true", "false"}
    ):
        raise DeployError(f"D3 container identity drifted before supervision: {name}")
    if fields[3] == "no":
        execute(("docker", "update", "--restart", "unless-stopped", name))
        fields = capture_output(inspect).split("\t")
        if len(fields) != 5 or fields[:4] != [
            expected_image, context.run_id, "host", "unless-stopped",
        ]:
            raise DeployError(f"D3 container supervision did not converge: {name}")
    if fields[4] == "false":
        execute(("docker", "start", name))


def container_images_by_name(context: D3Context, state: Mapping[str, Any]) -> dict[str, str]:
    containers = state["containers"]
    services = effective_service_images(context, state)
    result = {
        str(containers["runtime_api"]): services["runtime_api"],
        str(containers["integration_service"]): services["integration_service"],
        str(containers["sfcrm_app_service"]): services["sfcrm_app_service"],
    }
    if "mcp_worker" in context.images:
        result[str(containers["mcp_worker"])] = context.images["mcp_worker"]
    result.update({
        str(containers["workers"][stem]): effective_worker_image(context, state)
        for stem in context.worker_stems
    })
    return result


def effective_worker_image(context: D3Context, state: Mapping[str, Any]) -> str:
    """Resolve the one active Worker implementation without changing role policy."""
    image = state.get("workerImageOverride", context.images["runtime_worker"])
    if is_local_test3_bom(context.bom):
        require_local_image_context(context)
        if image != context.images["runtime_worker"] or not SHA256_RE.fullmatch(image):
            raise DeployError("local Test3 Worker identity cannot be overridden")
        return image
    repository = context.images["runtime_worker"].partition("@sha256:")[0]
    if not isinstance(image, str) or not re.fullmatch(
        re.escape(repository) + r"@sha256:[0-9a-f]{64}", image,
    ):
        raise DeployError("D3 Worker image override changed the owning repository")
    return image


def resolve_worker_runtime_config_image(
    context: D3Context,
    state: Mapping[str, Any],
    transition: Mapping[str, Any],
) -> tuple[str, str | None]:
    """Bind runtime tuning to the effective Worker image, including one legacy recovery."""
    effective = effective_worker_image(context, state)
    recorded = transition.get("runtimeWorkerImage")
    if recorded == effective:
        return effective, None
    base = context.images["runtime_worker"]
    if recorded == base and effective != base:
        return effective, base
    raise DeployError("L0 reconfiguration Worker image disagrees with active authority")


def sanitized_worker_image_lineage(state: Mapping[str, Any]) -> dict[str, Any]:
    """Return the last Worker implementation cutover without host paths or secrets."""
    operations = state.get("workerImageOperations")
    if not isinstance(operations, list) or not operations:
        raise DeployError("D3 state has no completed Worker image operation")
    operation = operations[-1]
    if (
        not isinstance(operation, dict)
        or operation.get("schema") != L0_WORKER_IMAGE_TRANSITION_SCHEMA
        or operation.get("result") != "activated"
    ):
        raise DeployError("D3 state has no activated Worker image lineage")
    paths = operation.get("previousRoleConfigs")
    digests = operation.get("previousRoleConfigDigests")
    stems = state_worker_stems(state)
    if (
        not isinstance(paths, dict) or set(paths) != set(stems)
        or not isinstance(digests, dict) or set(digests) != set(stems)
    ):
        raise DeployError("Worker image lineage lost its previous config seals")
    previous_sources: set[str] = set()
    previous_releases: set[str] = set()
    for stem in stems:
        path = Path(str(paths[stem]))
        if (
            not path.is_file() or path.is_symlink()
            or sha256_bytes(path.read_bytes()) != digests[stem]
        ):
            raise DeployError(f"Worker image lineage previous config changed: {stem}")
        config = load_json_object(path, f"previous Worker role config {stem}")
        source = str(config.get("worker_source_revision", ""))
        release = str(config.get("worker_release_digest", ""))
        if not re.fullmatch(r"[0-9a-f]{40}", source) or not SHA256_RE.fullmatch(release):
            raise DeployError(f"Worker image lineage previous identity is malformed: {stem}")
        previous_sources.add(source)
        previous_releases.add(release)
    return {
        "schema": "nexuscrm.serviceification-l0-worker-image-lineage/v1",
        "runId": state.get("runId"),
        "previousImage": operation.get("previousImage"),
        "previousSourceRevisions": sorted(previous_sources),
        "previousReleaseDigests": sorted(previous_releases),
        "candidateImage": operation.get("candidateImage"),
        "candidateSourceRevision": operation.get("candidateSourceRevision"),
        "activationReceiptCount": len(operation.get("activationOutputDigests", {})),
        "candidateConfigCount": len(operation.get("candidateConfigDigests", {})),
    }


def sealed_baseline_worker_identity(state: Mapping[str, Any]) -> tuple[str, str]:
    """Read the baseline Worker image/source from the run-owned resolved BOM."""
    run_root = Path(str(state.get("runRoot", ""))).expanduser()
    prepared = state.get("prepared")
    expected_digest = state.get("resolvedBomFileDigest")
    if (
        not run_root.is_absolute()
        or not isinstance(prepared, dict)
        or not isinstance(expected_digest, str)
        or not SHA256_RE.fullmatch(expected_digest)
    ):
        raise DeployError("D3 state has no sealed baseline Worker authority")
    raw_bom = Path(str(prepared.get("resolvedBomCopy", ""))).expanduser()
    expected_bom = (run_root / "resolved-bom.json").resolve()
    if (
        raw_bom.is_symlink()
        or raw_bom.resolve() != expected_bom
        or not expected_bom.is_file()
    ):
        raise DeployError("D3 baseline Worker BOM is not the sealed run-owned copy")
    serialized = expected_bom.read_bytes()
    if sha256_bytes(serialized) != expected_digest:
        raise DeployError("D3 baseline Worker BOM is not the sealed run-owned copy")
    try:
        resolved = parse_bom(serialized, str(expected_bom))
        if is_local_test3_bom(resolved):
            history = read_sealed_historical_bom(state)
            projected = history.bom
        else:
            projected = operational_bom(resolved)
        worker = projected["services"]["runtime_worker"]
        if is_local_test3_bom(resolved) and (
            state.get("envName") != "test-test3"
            or state.get("resolvedBomSchema") != LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION
            or any(state.get(key) != value for key, value in local_deployment_evidence(resolved).items())
        ):
            raise DeployError("local Worker baseline lost its Test3 authority")
        image = (
            history.images["runtime_worker"] if is_local_test3_bom(resolved)
            else publication_image(worker, "Runtime Worker")
        )
        source = worker["source_commit"]
    except (KeyError, TypeError, ValidationFailure) as exc:
        raise DeployError("D3 baseline Worker BOM has no exact Worker identity") from exc
    if not isinstance(source, str) or not re.fullmatch(r"[0-9a-f]{40}", source):
        raise DeployError("D3 baseline Worker source revision is malformed")
    return image, source


def sanitized_worker_serving_provenance(state: Mapping[str, Any]) -> dict[str, Any]:
    """Return the active Worker authority for both baseline and replacement deploys."""
    run_id = state.get("runId")
    images = state.get("images")
    exact_sources = state.get("exactSources")
    if (
        not isinstance(run_id, str) or not run_id
        or not isinstance(images, dict)
        or not isinstance(exact_sources, dict)
    ):
        raise DeployError("D3 state has no exact Worker serving authority")

    baseline_image, baseline_source = sealed_baseline_worker_identity(state)
    if images.get("runtime_worker") != baseline_image:
        raise DeployError("D3 state Worker image disagrees with its sealed BOM")
    serving_image = state.get("workerImageOverride", baseline_image)
    serving_source = state.get("workerImageSourceRevisionOverride", baseline_source)
    local = state.get("resolvedBomSchema") == LOCAL_TEST3_DEPLOYMENT_SCHEMA_VERSION
    image_pattern = (
        r"sha256:[0-9a-f]{64}" if local else
        r"ghcr\.io/pingcap-inc/nexuscrm-runtime-worker@sha256:[0-9a-f]{64}"
    )
    if local and (
        serving_image != baseline_image or serving_source != baseline_source
        or state.get("workerImageOperations")
    ):
        raise DeployError("local Test3 Worker identity cannot be overridden")
    if (
        not isinstance(baseline_image, str)
        or not re.fullmatch(
            image_pattern,
            baseline_image,
        )
        or not isinstance(serving_image, str)
        or not re.fullmatch(
            image_pattern,
            serving_image,
        )
        or not isinstance(baseline_source, str)
        or not re.fullmatch(r"[0-9a-f]{40}", baseline_source)
        or not isinstance(serving_source, str)
        or not re.fullmatch(r"[0-9a-f]{40}", serving_source)
    ):
        raise DeployError("D3 Worker serving identity is malformed")

    stems = state_worker_stems(state)
    families = state_worker_families(state)
    role_configs = state.get("workerRoleConfigDigests")
    ownership = state.get("workerOwnershipAfter")
    if (
        not isinstance(role_configs, dict) or set(role_configs) != set(stems)
        or not isinstance(ownership, dict)
        or set(ownership) not in READABLE_WORKER_FAMILY_SETS
    ):
        raise DeployError("D3 Worker serving evidence is incomplete")
    ownership = {**{family: None for family in SUPPORTED_WORKER_FAMILIES}, **ownership}
    if any(not SHA256_RE.fullmatch(str(role_configs[stem])) for stem in stems):
        raise DeployError("D3 Worker role config digest is malformed")

    release_digest = immutable_image_digest(serving_image)
    for family in families:
        receipt = ownership[family]
        if (
            not isinstance(receipt, dict)
            or receipt.get("workFamily") != family
            or receipt.get("active") is not True
            or receipt.get("workerReleaseDigest") != release_digest
        ):
            raise DeployError(f"D3 Worker serving receipt disagrees: {family}")

    operations = state.get("workerImageOperations")
    if operations is None:
        operations = []
    if not isinstance(operations, list):
        raise DeployError("D3 Worker image operation history has an unknown shape")

    result: dict[str, Any] = {
        "schema": "nexuscrm.serviceification-l0-worker-serving-provenance/v1",
        "runId": run_id,
        "lineageKind": "image-replacement" if operations else "resolved-bom-baseline",
        "servingImage": serving_image,
        "servingSourceRevision": serving_source,
        "workerRoleCount": len(stems),
        "workerRoleConfigDigests": {
            stem: role_configs[stem] for stem in stems
        },
        "ownershipReceiptCount": len(families),
        **({"releaseClass": LOCAL_TEST3_RELEASE_CLASS} if local else {}),
    }
    if operations:
        replacement = sanitized_worker_image_lineage(state)
        if (
            replacement.get("candidateImage") != serving_image
            or replacement.get("candidateSourceRevision") != serving_source
        ):
            raise DeployError("D3 Worker replacement lineage is not the serving authority")
        result["imageReplacement"] = replacement
    return result


def sanitized_runtime_worker_baseline_identity(
    state: Mapping[str, Any],
) -> dict[str, Any]:
    """Project the immutable D3 and database-shape facts used to compare runs."""
    if state.get("phase") != "serving":
        raise DeployError("Runtime Worker baseline requires an active serving state")
    run_id = state.get("runId")
    if not isinstance(run_id, str) or not RUN_ID_RE.fullmatch(run_id):
        raise DeployError("Runtime Worker baseline D3 run identity is malformed")
    worker_provenance = sanitized_worker_serving_provenance(state)

    image_roles = {
        "runtime_api", "runtime_worker", "integration_service",
        "sfcrm_app_service", "catalog_reconcile", "frontend",
    }
    if "mcp_worker" in state.get("images", {}):
        image_roles.add("mcp_worker")
    evidence = state.get("deploymentEvidence")
    catalog = state.get("catalogAfter")
    if (
        not isinstance(evidence, dict)
        or evidence.get("schema") != EVIDENCE_SCHEMA
        or evidence.get("stage") != "serving-verified"
        or evidence.get("runId") != run_id
        or not isinstance(catalog, dict)
    ):
        raise DeployError("Runtime Worker baseline has no verified deployment authority")

    resolved_digest = state.get("resolvedBomFileDigest")
    desired_digest = state.get("desiredBomDigest")
    if (
        not isinstance(resolved_digest, str)
        or not SHA256_RE.fullmatch(resolved_digest)
        or not isinstance(desired_digest, str)
        or not SHA256_RE.fullmatch(desired_digest)
        or evidence.get("resolvedBomFileDigest") != resolved_digest
        or evidence.get("desiredBomDigest") != desired_digest
    ):
        raise DeployError("Runtime Worker baseline BOM identity is incomplete")
    prepared = state.get("prepared")
    if not isinstance(prepared, dict):
        raise DeployError("Runtime Worker baseline has no prepared BOM authority")
    resolved_path = Path(str(prepared.get("resolvedBomCopy", "")))
    active_bom = read_sealed_historical_bom(state).bom
    if active_bom.get("desired_bom_sha256") != desired_digest:
        raise DeployError("Runtime Worker baseline resolved BOM authority disagrees")
    effective_policies = effective_runtime_worker_role_policies(state, active_bom)
    effective_policy_digest = sha256_bytes(canonical_bytes(effective_policies))

    evidence_images = evidence.get("images")
    base_images = state.get("images")
    local = is_local_test3_bom(active_bom)
    if (
        not isinstance(evidence_images, dict)
        or set(evidence_images) != image_roles
        or not isinstance(base_images, dict)
        or set(base_images) != image_roles
        or any(
            not isinstance(image, str)
            or not re.fullmatch(
                r"sha256:[0-9a-f]{64}" if local else
                r"ghcr[.]io/pingcap-inc/[a-z0-9._/-]+@sha256:[0-9a-f]{64}",
                image,
            )
            for image in (*evidence_images.values(), *base_images.values())
        )
    ):
        raise DeployError("Runtime Worker baseline image identity is incomplete")
    service_overrides = state.get("serviceImageOverrides", {})
    service_roles = {"runtime_api", "integration_service", "sfcrm_app_service"}
    if (
        not isinstance(service_overrides, dict)
        or not set(service_overrides).issubset(service_roles)
    ):
        raise DeployError("Runtime Worker baseline service overrides are malformed")
    active_images = dict(base_images)
    for role, image in service_overrides.items():
        repository = str(base_images[role]).partition("@sha256:")[0]
        if not isinstance(image, str) or not re.fullmatch(
            re.escape(repository) + r"@sha256:[0-9a-f]{64}", image,
        ):
            raise DeployError("Runtime Worker baseline service override changed owner")
        active_images[role] = image
    active_images["runtime_worker"] = worker_provenance["servingImage"]
    frontend_override = state.get("frontendImageOverride")
    if frontend_override is not None:
        repository = str(base_images["frontend"]).partition("@sha256:")[0]
        if not isinstance(frontend_override, str) or not re.fullmatch(
            re.escape(repository) + r"@sha256:[0-9a-f]{64}", frontend_override,
        ):
            raise DeployError("Runtime Worker baseline frontend override changed owner")
        active_images["frontend"] = frontend_override

    integer_facts = {
        "catalogRevision": evidence.get("catalogRevision"),
        "securityEpoch": evidence.get("securityEpoch"),
        "schemaGeneration": catalog.get("schemaGeneration"),
        "engineGeneration": catalog.get("engineGeneration"),
    }
    if any(
        not isinstance(value, int) or isinstance(value, bool) or value < 0
        for value in integer_facts.values()
    ) or any(
        integer_facts[field] <= 0
        for field in ("schemaGeneration", "engineGeneration")
    ):
        raise DeployError("Runtime Worker baseline database shape is incomplete")
    if (
        catalog.get("catalogRevision") != integer_facts["catalogRevision"]
        or catalog.get("securityEpoch") != integer_facts["securityEpoch"]
    ):
        raise DeployError("Runtime Worker baseline deployment and Catalog disagree")

    frontend_digest = evidence.get("frontendTargetDigest")
    if not isinstance(frontend_digest, str) or not SHA256_RE.fullmatch(frontend_digest):
        raise DeployError("Runtime Worker baseline frontend identity is incomplete")
    if (
        evidence.get("containerCount") != len(state_worker_stems(state)) + 3 + int("mcp_worker" in image_roles)
        or evidence.get("runtimeEndpointCount") != 1
        or evidence.get("workerArtifactCount") != 1
        or evidence.get("workerRoleCount") != len(state_worker_stems(state))
        or evidence.get("legacyProcessCount") != 0
        or evidence.get("gatewayArtifactCount") != 0
        or evidence.get("fallbackPathCount") != 0
    ):
        raise DeployError("Runtime Worker baseline topology is not the exact D3 shape")

    return {
        "schema": "nexuscrm.runtime-worker-baseline-identity/v1",
        "deploymentRunId": run_id,
        "resolvedBomFileDigest": resolved_digest,
        "desiredBomDigest": desired_digest,
        **integer_facts,
        "images": {role: active_images[role] for role in sorted(image_roles)},
        "workerServingSourceRevision": worker_provenance["servingSourceRevision"],
        "effectiveWorkerRolePolicyDigest": effective_policy_digest,
        "frontendTargetDigest": frontend_digest,
        "runtimeEndpointCount": 1,
        "workerArtifactCount": 1,
        "workerRoleCount": len(state_worker_stems(state)),
    }


def effective_service_images(
    context: D3Context, state: Mapping[str, Any],
) -> dict[str, str]:
    """Resolve serving implementation overrides without changing release metadata."""
    roles = {"runtime_api", "integration_service", "sfcrm_app_service"}
    overrides = state.get("serviceImageOverrides", {})
    if not isinstance(overrides, dict) or not set(overrides).issubset(roles):
        raise DeployError("D3 serving state has an invalid service image override map")
    result = {role: context.images[role] for role in roles}
    for role, image in overrides.items():
        if not isinstance(image, str):
            raise DeployError("D3 service image override is not an immutable reference")
        repository = context.images[role].partition("@sha256:")[0]
        if not re.fullmatch(re.escape(repository) + r"@sha256:[0-9a-f]{64}", image):
            raise DeployError("D3 service image override changed the role repository")
        result[role] = image
    return result


def serving_health_targets(
    context: D3Context, state: Mapping[str, Any],
) -> list[tuple[str, str]]:
    names = expected_serving_container_order(state)
    targets = [(names[0], f"http://127.0.0.1:{context.ports.runtime_admin}/readyz")]
    targets.extend((
        names[index + 1],
        f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz",
    ) for index, stem in enumerate(state_worker_stems(state)))
    targets.extend((
        (str(state["containers"]["integration_service"]), f"http://127.0.0.1:{context.ports.integration_http}/readyz"),
        (str(state["containers"]["sfcrm_app_service"]), f"http://127.0.0.1:{context.ports.sfcrm_http}/readyz"),
    ))
    if "mcp_worker" in context.images:
        targets.append((str(state["containers"]["mcp_worker"]), f"http://127.0.0.1:{context.ports.mcp_http}/readyz"))
    return targets


def runtime_docker_health_targets(
    context: D3Context, state: Mapping[str, Any],
) -> list[tuple[str, str]]:
    names = expected_serving_container_order(state)
    targets = [(names[0], f"http://127.0.0.1:{context.ports.runtime_admin}/readyz")]
    targets.extend((
        names[index + 1],
        f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz",
    ) for index, stem in enumerate(state_worker_stems(state)))
    return targets


def wait_for_serving_health(
    context: D3Context,
    state: Mapping[str, Any],
    *,
    wait_ready: Callable[[str, str], None],
    wait_healthy: Callable[[str, str], None],
) -> int:
    for name, url in serving_health_targets(context, state):
        wait_ready(url, name)
    docker_targets = runtime_docker_health_targets(context, state)
    for name, url in docker_targets:
        wait_healthy(url, name)
    return len(docker_targets)


def recover_incomplete_serving_inventory(
    context: D3Context, state: Mapping[str, Any],
) -> bool:
    """Return a sealed serving deployment to its idempotent start phase.

    A host restart normally leaves stopped containers behind, which
    ``verify_serving`` can start in place. An interrupted operator cleanup can
    instead remove only part of the exact tuple. The serving artifacts and
    container identities are already sealed in state, so the safe recovery is
    to re-enter ``serving-start-in-progress`` and let ``start_serving`` rebuild
    that same tuple. Containers outside the sealed inventory remain a hard
    ownership conflict.
    """
    expected = set(expected_serving_container_order(state))
    present = set(owned_containers())
    unexpected = present - expected
    if unexpected:
        raise DeployError(
            "D3 serving container inventory contains another owner: "
            + ", ".join(sorted(unexpected))
        )
    missing = expected - present
    if not missing:
        return False
    replace_state(
        context,
        expected_phase="serving",
        updates={
            "phase": "serving-start-in-progress",
            "servingInventoryRecovery": {
                "schema": "nexuscrm.serviceification-serving-inventory-recovery/v1",
                "missingContainers": sorted(missing),
            },
        },
    )
    return True


def verify_serving(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
    wait_healthy: Callable[[str, str], None] = wait_docker_healthy,
) -> dict[str, Any]:
    state = require_state(
        context,
        expected_phase="serving",
        allow_legacy_reset_receipt=True,
    )
    authority = verify_catalog_after(context, state)
    ownership = require_worker_ownership_after(state)
    if read_worker_ownership(context.envv) != ownership:
        raise DeployError("Runtime Worker ownership changed while D3 was serving")
    expected_order = expected_serving_container_order(state)
    if set(owned_containers()) != set(expected_order):
        raise DeployError("D3 serving container inventory is incomplete or contains another owner")
    expected_images = container_images_by_name(context, state)
    for name in expected_order:
        converge_container_supervision(
            context, name, expected_images[name],
            execute=execute, capture_output=capture_output,
        )
        if not verify_container_identity(
            context, name, expected_images[name], capture_output=capture_output,
        ):
            raise DeployError(f"D3 serving container is stopped: {name}")
    docker_healthy_container_count = wait_for_serving_health(
        context,
        state,
        wait_ready=wait_ready,
        wait_healthy=wait_healthy,
    )
    legacy_units = state.get("previousSystemdUnits")
    if not isinstance(legacy_units, dict):
        raise DeployError("D3 state has no legacy unit inventory")
    active_legacy = [
        unit for unit in legacy_units
        if staging.systemd_unit_state(unit).get("active_state") == "active"
    ]
    if active_legacy:
        raise DeployError("legacy serving units are active beside the D3 tuple")
    state = ensure_d3_public_ingress(context)
    frontend = Path(str(state["publishedFrontendTarget"])).resolve()
    if resolve_frontend_dist_target(runtime_paths(context.env_root)["frontend_dist"]) != frontend:
        raise DeployError("test2 ingress is not serving the exact D3 frontend")
    if any("gateway" in name.lower() or "gateway" in image.lower() for name, image in expected_images.items()):
        raise DeployError("D3 serving inventory contains a forbidden generic Gateway")
    mcp_evidence = None
    if "mcp_worker" in context.images:
        from verify_hosted_mcp import probe
        mcp_evidence = probe(context.envv["PUBLIC_BASE_URL"])
        if not mcp_evidence["entryPassed"]:
            raise DeployError("public MCP discovery/challenge failed after target cutover")
    evidence = {
        "schema": EVIDENCE_SCHEMA,
        "stage": "serving-verified",
        **({"mcpEntry": mcp_evidence} if mcp_evidence is not None else {}),
        **local_deployment_evidence(context.bom),
        "runId": context.run_id,
        "resolvedBomFileDigest": context.bom_file_digest,
        "desiredBomDigest": context.bom["desired_bom_sha256"],
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
        "images": {
            **dict(context.images),
            **effective_service_images(context, state),
            "runtime_worker": effective_worker_image(context, state),
            **(
                {"frontend": state["frontendImageOverride"]}
                if isinstance(state.get("frontendImageOverride"), str)
                else {}
            ),
        },
        "containerCount": len(expected_order),
        "dockerHealthyContainerCount": docker_healthy_container_count,
        "runtimeEndpointCount": 1,
        "workerArtifactCount": 1,
        "workerRoleCount": len(context.worker_stems),
        "legacyProcessCount": 0,
        "gatewayArtifactCount": 0,
        "fallbackPathCount": 0,
        "governedReferenceData": state.get("governedReferenceData"),
        "runtimeAppShell": state.get("runtimeAppShell"),
        "frontendTargetDigest": sha256_bytes(
            (frontend / "version.json").read_bytes(),
        ),
        "rollbackRehearsalCount": state.get("rollbackRehearsalCount", 0),
    }
    replace_state(
        context, expected_phase="serving",
        updates={"deploymentEvidence": evidence},
    )
    return evidence


def converge_worker_image_schema(
    context: D3Context,
    state: Mapping[str, Any],
    runtime_worker_image: str,
    worker_source_revision: str,
    schema_carrier_image: str,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
) -> str:
    """Converge additive L0 schema from the exact Worker source before cutover."""
    worker_repository = context.images["runtime_worker"].partition("@sha256:")[0]
    schema_repository = context.images["runtime_api"].partition("@sha256:")[0]
    if not re.fullmatch(
        re.escape(worker_repository) + r"@sha256:[0-9a-f]{64}",
        runtime_worker_image,
    ):
        raise DeployError("Runtime Worker schema convergence received an invalid Worker image")
    if not re.fullmatch(
        re.escape(schema_repository) + r"@sha256:[0-9a-f]{64}",
        schema_carrier_image,
    ):
        raise DeployError(
            "Runtime Worker schema carrier must use the Runtime API repository and exact digest",
        )
    if not re.fullmatch(r"[0-9a-f]{40}", worker_source_revision):
        raise DeployError("Runtime Worker schema convergence has no exact Worker source revision")

    catalog_artifacts = state.get("catalogArtifacts")
    if not isinstance(catalog_artifacts, dict):
        raise DeployError("Runtime Worker schema convergence lost Catalog artifacts")
    db_env_file = Path(str(catalog_artifacts.get("databaseEnv", "")))
    if (
        not db_env_file.is_file()
        or db_env_file.is_symlink()
        or db_env_file.stat().st_mode & 0o077
    ):
        raise DeployError("Catalog database credential file is missing or disclosed")

    execute(("docker", "pull", "--quiet", schema_carrier_image))
    schema_source_revision = capture_output((
        "docker", "image", "inspect", "--format",
        '{{index .Config.Labels "org.opencontainers.image.revision"}}',
        schema_carrier_image,
    )).strip()
    if schema_source_revision != worker_source_revision:
        raise DeployError(
            "Runtime Worker and L0 schema carrier must have the same source revision",
        )

    execute((
        "docker", "run", "--rm",
        "--name", container_name(
            context.run_id,
            f"worker-schema-{new_uuid7()[:12]}",
        ),
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
        "--env-file", str(db_env_file),
        "--entrypoint", "/usr/local/bin/l0-controlplane",
        schema_carrier_image,
        "ensure-l0-schema-ddl",
    ))
    return schema_source_revision


def supersede_stopped_worker_image_candidate(
    context: D3Context,
    state: dict[str, Any],
    transition: dict[str, Any],
    runtime_worker_image: str,
    schema_carrier_image: str,
    *,
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
) -> tuple[dict[str, Any], dict[str, Any]]:
    """Replace a failed, fully stopped Worker candidate without weakening ownership."""
    if state.get("phase") != "worker-image-replace-in-progress":
        raise DeployError("Runtime Worker candidate supersede requires a stopped replacement")
    previous_candidate = str(transition.get("candidateImage", ""))
    if runtime_worker_image == previous_candidate:
        return state, transition

    containers = state.get("containers")
    if not isinstance(containers, dict) or not isinstance(containers.get("workers"), dict):
        raise DeployError("Runtime Worker candidate supersede lost the serving inventory")
    worker_names = {str(containers["workers"][stem]) for stem in context.worker_stems}
    if worker_names & set(owned_containers()):
        raise DeployError("Runtime Worker candidate supersede requires every Worker to be stopped")

    authority = verify_catalog_after(context, state)
    if (
        authority.catalog_revision != transition.get("catalogRevision")
        or authority.security_epoch != transition.get("securityEpoch")
    ):
        raise DeployError("Catalog changed before Runtime Worker candidate supersede")
    old_activation_dir = Path(str(transition.get("activationConfig", "")))
    old_output_dir = Path(str(transition.get("activationOutput", "")))
    old_request_digests = transition.get("activationConfigDigests")
    if not isinstance(old_request_digests, dict) or set(old_request_digests) != set(context.worker_stems):
        raise DeployError("stopped Worker candidate lost sealed activation requests")
    ownership: dict[str, WorkerOwnership | None] = {
        family: None for family in SUPPORTED_WORKER_FAMILIES
    }
    old_outputs: dict[str, dict[str, Any]] = {}
    for stem in context.worker_stems:
        request_path = old_activation_dir / f"{stem}.json"
        output_path = old_output_dir / f"{stem}.json"
        if (
            not request_path.is_file()
            or request_path.is_symlink()
            or sha256_bytes(request_path.read_bytes()) != old_request_digests[stem]
            or not output_path.is_file()
            or output_path.is_symlink()
        ):
            raise DeployError(f"stopped Worker candidate lost sealed ownership evidence: {stem}")
        request = load_json_object(request_path, f"stopped Worker activation request {stem}")
        output = load_json_object(output_path, f"stopped Worker activation output {stem}")
        if (
            output.get("schema") != "nexuscrm.runtime-work-ownership-activation-result/v1"
            or output.get("operationId") != request.get("operationId")
            or output.get("workFamily") != request.get("workFamily")
            or output.get("deploymentId") != request.get("deploymentId")
            or output.get("workerReleaseDigest") != request.get("workerReleaseDigest")
            or output.get("ownershipEpoch")
            != str(int(str(request.get("expectedOwnershipEpoch"))) + 1)
        ):
            raise DeployError(f"stopped Worker ownership receipt disagrees with its request: {stem}")
        family = str(output["workFamily"])
        old_outputs[stem] = output
        ownership[family] = WorkerOwnership(
            work_family=family,
            deployment_id=str(output["deploymentId"]),
            worker_release_digest=str(output["workerReleaseDigest"]),
            ownership_epoch=int(str(output["ownershipEpoch"])),
            active=True,
            operation_id=str(output["operationId"]),
        )
    if read_worker_ownership(context.envv) != ownership:
        raise DeployError("Runtime Worker ownership changed before candidate supersede")

    previous_candidate_dir = Path(str(transition.get("candidateConfig", "")))
    active_paths = transition.get("previousRoleConfigs")
    active_digests = transition.get("previousRoleConfigDigests")
    candidate_seals = transition.get("candidateConfigDigests")
    if (
        not isinstance(active_paths, dict)
        or not isinstance(active_digests, dict)
        or (candidate_seals is not None and not isinstance(candidate_seals, dict))
    ):
        raise DeployError("stopped Worker candidate lost role-config authority")
    previous_paths: dict[str, str] = {}
    previous_digests: dict[str, str] = {}
    for stem in context.worker_stems:
        family = f"runtime.{stem}.v1"
        current = ownership.get(family)
        path = previous_candidate_dir / f"{stem}.json"
        if current is None or not path.is_file() or path.is_symlink():
            raise DeployError(f"stopped Worker candidate config is unavailable: {stem}")
        active_path = Path(str(active_paths.get(stem, "")))
        if (
            not active_path.is_file()
            or active_path.is_symlink()
            or sha256_bytes(active_path.read_bytes()) != active_digests.get(stem)
        ):
            raise DeployError(f"pre-candidate Worker config changed: {stem}")
        expected_config = load_json_object(active_path, f"pre-candidate Worker config {stem}")
        expected_config.update({
            "deployment_id": old_outputs[stem]["deploymentId"],
            "worker_release_digest": old_outputs[stem]["workerReleaseDigest"],
            "worker_source_revision": transition["candidateSourceRevision"],
            "ownership_epoch": int(str(old_outputs[stem]["ownershipEpoch"])),
        })
        if path.read_bytes() != canonical_bytes(expected_config):
            raise DeployError(f"stopped Worker candidate config is not derived from sealed inputs: {stem}")
        if candidate_seals is not None and sha256_bytes(path.read_bytes()) != candidate_seals.get(stem):
            raise DeployError(f"stopped Worker candidate config seal changed: {stem}")
        config = load_json_object(path, f"stopped Worker candidate config {stem}")
        if (
            config.get("work_family") != family
            or config.get("deployment_id") != current.deployment_id
            or config.get("worker_release_digest") != current.worker_release_digest
            or config.get("ownership_epoch") != current.ownership_epoch
            or current.worker_release_digest != previous_candidate.partition("@")[2]
            or config.get("worker_source_revision") != transition.get("candidateSourceRevision")
        ):
            raise DeployError(f"stopped Worker candidate disagrees with ownership: {stem}")
        previous_paths[stem] = str(path)
        previous_digests[stem] = sha256_bytes(path.read_bytes())

    repository = context.images["runtime_worker"].partition("@sha256:")[0]
    if not re.fullmatch(re.escape(repository) + r"@sha256:[0-9a-f]{64}", runtime_worker_image):
        raise DeployError("Runtime Worker supersede must use the owning repository and exact digest")
    execute(("docker", "pull", "--quiet", runtime_worker_image))
    source_revision = capture_output((
        "docker", "image", "inspect", "--format",
        '{{index .Config.Labels "org.opencontainers.image.revision"}}',
        runtime_worker_image,
    )).strip()
    if not re.fullmatch(r"[0-9a-f]{40}", source_revision):
        raise DeployError("Runtime Worker supersede image has no exact OCI source revision")
    schema_source_revision = converge_worker_image_schema(
        context,
        state,
        runtime_worker_image,
        source_revision,
        schema_carrier_image,
        execute=execute,
        capture_output=capture_output,
    )

    operation_id = new_uuid7()
    operation_root = context.run_root / "worker-image" / operation_id
    activation_dir = operation_root / "activation-config"
    output_dir = operation_root / "activation-output"
    candidate_dir = operation_root / "candidate-config"
    for path in (activation_dir, output_dir, candidate_dir):
        path.mkdir(parents=True, mode=0o700, exist_ok=False)
    release_digest = runtime_worker_image.partition("@")[2]
    request_digests: dict[str, str] = {}
    for stem in context.worker_stems:
        family = f"runtime.{stem}.v1"
        current = ownership.get(family)
        if current is None:
            raise DeployError(f"Runtime Worker ownership is inactive: {family}")
        request = {
            "schema": context.bom["execution"]["runtime_work_activation_schema"],
            "operationId": f"{operation_id}-activate-{stem}",
            "workFamily": family,
            "deploymentId": f"d3-test2-{context.run_id}-worker-image-{operation_id[:12]}-{stem}",
            "workerReleaseDigest": release_digest,
            "expectedOwnershipEpoch": str(current.ownership_epoch),
        }
        request_path = activation_dir / f"{stem}.json"
        write_protected_json(request_path, request)
        request_digests[stem] = sha256_bytes(request_path.read_bytes())

    superseded = {
        "operationId": transition.get("operationId"),
        "candidateImage": previous_candidate,
        "candidateSourceRevision": transition.get("candidateSourceRevision"),
    }
    replacement = {
        "schema": L0_WORKER_IMAGE_TRANSITION_SCHEMA,
        "operationId": operation_id,
        "previousImage": previous_candidate,
        "candidateImage": runtime_worker_image,
        "candidateSourceRevision": source_revision,
        "schemaCarrierImage": schema_carrier_image,
        "schemaCarrierSourceRevision": schema_source_revision,
        "schemaConvergenceCommand": "ensure-l0-schema-ddl",
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
        "previousOwnership": worker_ownership_payload(ownership),
        "previousRoleConfigDigests": previous_digests,
        "previousRoleConfigs": previous_paths,
        "activationConfig": str(activation_dir),
        "activationConfigDigests": request_digests,
        "activationOutput": str(output_dir),
        "candidateConfig": str(candidate_dir),
        "supersededStoppedCandidate": superseded,
    }
    state = replace_state(
        context,
        expected_phase="worker-image-replace-in-progress",
        updates={"workerImageTransition": replacement},
    )
    return state, replacement


def replace_worker_image(
    context: D3Context,
    runtime_worker_image: str,
    schema_carrier_image: str,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    capture_diagnostic: Callable[[Sequence[str]], str] = default_capture_combined,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
) -> dict[str, Any]:
    """Move every selected Worker family to one exact implementation and ownership tuple."""
    state = require_state(
        context, expected_phase={"serving", "worker-image-replace-in-progress"},
    )
    repository = context.images["runtime_worker"].partition("@sha256:")[0]
    if not re.fullmatch(re.escape(repository) + r"@sha256:[0-9a-f]{64}", runtime_worker_image):
        raise DeployError("Runtime Worker replacement must use the owning repository and exact digest")
    current_image = effective_worker_image(context, state)
    transition = state.get("workerImageTransition")

    if state["phase"] == "serving":
        if runtime_worker_image == current_image:
            return {
                "schema": EVIDENCE_SCHEMA,
                "stage": "worker-image-unchanged",
                "runId": context.run_id,
                "image": runtime_worker_image,
            }
        authority = verify_catalog_after(context, state)
        ownership = require_worker_ownership_after(state)
        if read_worker_ownership(context.envv) != ownership:
            raise DeployError("Runtime Worker ownership changed before image replacement")
        role_digests = verify_worker_role_config_seals(state)
        expected_order = expected_serving_container_order(state)
        if set(owned_containers()) != set(expected_order):
            raise DeployError("Worker image replacement requires the complete serving inventory")
        expected_images = container_images_by_name(context, state)
        for name in expected_order:
            if not verify_container_identity(
                context, name, expected_images[name], capture_output=capture_output,
            ):
                raise DeployError(f"Worker image replacement requires a running exact container: {name}")

        execute(("docker", "pull", "--quiet", runtime_worker_image))
        source_revision = capture_output((
            "docker", "image", "inspect", "--format",
            '{{index .Config.Labels "org.opencontainers.image.revision"}}',
            runtime_worker_image,
        )).strip()
        if not re.fullmatch(r"[0-9a-f]{40}", source_revision):
            raise DeployError("Runtime Worker image has no exact OCI source revision")

        schema_source_revision = converge_worker_image_schema(
            context,
            state,
            runtime_worker_image,
            source_revision,
            schema_carrier_image,
            execute=execute,
            capture_output=capture_output,
        )

        operation_id = new_uuid7()
        operation_root = context.run_root / "worker-image" / operation_id
        activation_dir = operation_root / "activation-config"
        output_dir = operation_root / "activation-output"
        candidate_dir = operation_root / "candidate-config"
        for path in (activation_dir, output_dir, candidate_dir):
            path.mkdir(parents=True, mode=0o700, exist_ok=False)
        release_digest = runtime_worker_image.partition("@")[2]
        request_digests: dict[str, str] = {}
        previous_configs: dict[str, str] = {}
        for stem in context.worker_stems:
            family = f"runtime.{stem}.v1"
            before = ownership.get(family)
            if before is None:
                raise DeployError(f"Runtime Worker ownership is inactive: {family}")
            current_path = worker_role_config_path(state, stem)
            current_config = load_json_object(current_path, f"active Worker role config {stem}")
            if (
                current_config.get("work_family") != family
                or current_config.get("deployment_id") != before.deployment_id
                or current_config.get("worker_release_digest") != before.worker_release_digest
                or current_config.get("ownership_epoch") != before.ownership_epoch
                or not re.fullmatch(r"[0-9a-f]{40}", str(current_config.get("worker_source_revision", "")))
            ):
                raise DeployError(f"active Worker role config disagrees with ownership: {stem}")
            previous_configs[stem] = str(current_path)
            request = {
                "schema": context.bom["execution"]["runtime_work_activation_schema"],
                "operationId": f"{operation_id}-activate-{stem}",
                "workFamily": family,
                "deploymentId": f"d3-test2-{context.run_id}-worker-image-{operation_id[:12]}-{stem}",
                "workerReleaseDigest": release_digest,
                "expectedOwnershipEpoch": str(before.ownership_epoch),
            }
            request_path = activation_dir / f"{stem}.json"
            write_protected_json(request_path, request)
            request_digests[stem] = sha256_bytes(request_path.read_bytes())

        transition = {
            "schema": L0_WORKER_IMAGE_TRANSITION_SCHEMA,
            "operationId": operation_id,
            "previousImage": current_image,
            "candidateImage": runtime_worker_image,
            "candidateSourceRevision": source_revision,
            "schemaCarrierImage": schema_carrier_image,
            "schemaCarrierSourceRevision": schema_source_revision,
            "schemaConvergenceCommand": "ensure-l0-schema-ddl",
            "catalogRevision": authority.catalog_revision,
            "securityEpoch": authority.security_epoch,
            "previousOwnership": worker_ownership_payload(ownership),
            "previousRoleConfigDigests": role_digests,
            "previousRoleConfigs": previous_configs,
            "activationConfig": str(activation_dir),
            "activationConfigDigests": request_digests,
            "activationOutput": str(output_dir),
            "candidateConfig": str(candidate_dir),
        }
        state = replace_state(
            context, expected_phase="serving", updates={
                "phase": "worker-image-replace-in-progress",
                "workerImageTransition": transition,
            },
        )

    if (
        state["phase"] == "worker-image-replace-in-progress"
        and isinstance(transition, dict)
        and transition.get("candidateImage") != runtime_worker_image
    ):
        state, transition = supersede_stopped_worker_image_candidate(
            context,
            state,
            transition,
            runtime_worker_image,
            schema_carrier_image,
            execute=execute,
            capture_output=capture_output,
        )

    if (
        not isinstance(transition, dict)
        or transition.get("schema") != L0_WORKER_IMAGE_TRANSITION_SCHEMA
        or transition.get("candidateImage") != runtime_worker_image
        or not re.fullmatch(
            r"[0-9a-f]{40}", str(transition.get("candidateSourceRevision", "")),
        )
    ):
        raise DeployError("another Runtime Worker image replacement is already active")
    if transition.get("schemaCarrierImage") is None:
        schema_source_revision = converge_worker_image_schema(
            context,
            state,
            runtime_worker_image,
            str(transition["candidateSourceRevision"]),
            schema_carrier_image,
            execute=execute,
            capture_output=capture_output,
        )
        transition = {
            **transition,
            "schemaCarrierImage": schema_carrier_image,
            "schemaCarrierSourceRevision": schema_source_revision,
            "schemaConvergenceCommand": "ensure-l0-schema-ddl",
        }
        state = replace_state(
            context,
            expected_phase="worker-image-replace-in-progress",
            updates={"workerImageTransition": transition},
        )
    elif (
        transition.get("schemaCarrierImage") != schema_carrier_image
        or transition.get("schemaCarrierSourceRevision")
        != transition.get("candidateSourceRevision")
        or transition.get("schemaConvergenceCommand") != "ensure-l0-schema-ddl"
    ):
        raise DeployError("Runtime Worker recovery changed its sealed L0 schema carrier")
    authority = verify_catalog_after(context, state)
    if (
        authority.catalog_revision != transition.get("catalogRevision")
        or authority.security_epoch != transition.get("securityEpoch")
    ):
        raise DeployError("Catalog changed during Runtime Worker image replacement")

    containers = state.get("containers")
    catalog_artifacts = state.get("catalogArtifacts")
    if not isinstance(containers, dict) or not isinstance(catalog_artifacts, dict):
        raise DeployError("Worker image replacement lost serving artifacts")
    activation_dir = Path(str(transition["activationConfig"]))
    output_dir = Path(str(transition["activationOutput"]))
    candidate_dir = Path(str(transition["candidateConfig"]))
    request_digests = transition.get("activationConfigDigests")
    if not isinstance(request_digests, dict) or set(request_digests) != set(context.worker_stems):
        raise DeployError("Worker image replacement lost sealed activation requests")
    requests: dict[str, dict[str, Any]] = {}
    for stem in context.worker_stems:
        path = activation_dir / f"{stem}.json"
        if not path.is_file() or path.is_symlink() or sha256_bytes(path.read_bytes()) != request_digests[stem]:
            raise DeployError(f"sealed Worker activation request changed: {stem}")
        requests[stem] = load_json_object(path, f"Worker activation request {stem}")

    expected_order = expected_serving_container_order(state)
    worker_names = {str(containers["workers"][stem]) for stem in context.worker_stems}
    present = set(owned_containers())
    if not present.issubset(set(expected_order)):
        raise DeployError("Worker image replacement found a container outside the serving inventory")
    non_worker_names = set(expected_order) - worker_names
    if not non_worker_names.issubset(present):
        raise DeployError("Worker image replacement lost a non-Worker serving container")
    expected_non_workers = container_images_by_name(context, state)
    for name in non_worker_names:
        if not verify_container_identity(
            context, name, expected_non_workers[name], capture_output=capture_output,
        ):
            raise DeployError(f"non-Worker serving container changed during Worker cutover: {name}")
    allowed_worker_images = {str(transition["previousImage"]), runtime_worker_image}
    for name in worker_names & present:
        fields = capture_output((
            "docker", "inspect", "--format",
            '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}\t'
            '{{.HostConfig.NetworkMode}}\t{{.HostConfig.RestartPolicy.Name}}\t{{.State.Running}}',
            name,
        )).split("\t")
        if (
            len(fields) != 5 or fields[0] not in allowed_worker_images
            or fields[1:4] != [context.run_id, "host", "unless-stopped"]
            or fields[4] not in {"true", "false"}
        ):
            raise DeployError(f"Worker cutover found an unknown implementation: {name}")
        execute(("docker", "rm", "-f", name))

    db_env_file = Path(str(catalog_artifacts.get("databaseEnv", "")))
    if not db_env_file.is_file() or db_env_file.is_symlink() or db_env_file.stat().st_mode & 0o077:
        raise DeployError("Catalog database credential file is missing or disclosed")
    activation_outputs: dict[str, dict[str, Any]] = {}
    for stem in context.worker_stems:
        output = output_dir / f"{stem}.json"
        if not output.exists() and not output.is_symlink():
            execute((
                "docker", "run", "--rm",
                "--name", container_name(context.run_id, f"worker-image-activation-{stem}"),
                "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
                "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
                "--env-file", str(db_env_file),
                "--mount", f"type=bind,src={activation_dir / f'{stem}.json'},dst=/etc/nexuscrm/work-ownership.json,readonly",
                "--mount", f"type=bind,src={output_dir},dst=/output",
                context.images["catalog_reconcile"],
                "work-ownership", "-config", "/etc/nexuscrm/work-ownership.json",
                "-output", f"/output/{stem}.json",
            ))
        if not output.is_file() or output.is_symlink():
            raise DeployError(f"Worker activation emitted no regular receipt: {stem}")
        activation = load_json_object(output, f"Worker activation output {stem}")
        request = requests[stem]
        if set(activation) != {
            "schema", "operationId", "workFamily", "deploymentId", "workerReleaseDigest", "ownershipEpoch",
        } or (
            activation.get("schema") != "nexuscrm.runtime-work-ownership-activation-result/v1"
            or activation.get("operationId") != request.get("operationId")
            or activation.get("workFamily") != request.get("workFamily")
            or activation.get("deploymentId") != request.get("deploymentId")
            or activation.get("workerReleaseDigest") != request.get("workerReleaseDigest")
            or activation.get("ownershipEpoch") != str(int(str(request.get("expectedOwnershipEpoch"))) + 1)
        ):
            raise DeployError(f"Worker activation receipt disagrees with its exact request: {stem}")
        os.chmod(output, 0o400)
        activation_outputs[stem] = activation

    expected_after: dict[str, WorkerOwnership | None] = {
        family: None for family in SUPPORTED_WORKER_FAMILIES
    }
    candidate_paths: dict[str, str] = {}
    candidate_digests: dict[str, str] = {}
    previous_paths = transition.get("previousRoleConfigs")
    previous_digests = transition.get("previousRoleConfigDigests")
    if not isinstance(previous_paths, dict) or not isinstance(previous_digests, dict):
        raise DeployError("Worker image replacement lost previous config authority")
    for stem in context.worker_stems:
        activation = activation_outputs[stem]
        family = str(activation["workFamily"])
        epoch = int(str(activation["ownershipEpoch"]))
        expected_after[family] = WorkerOwnership(
            work_family=family,
            deployment_id=str(activation["deploymentId"]),
            worker_release_digest=str(activation["workerReleaseDigest"]),
            ownership_epoch=epoch,
            active=True,
            operation_id=str(activation["operationId"]),
        )
        previous_path = Path(str(previous_paths.get(stem, "")))
        if (
            not previous_path.is_file() or previous_path.is_symlink()
            or sha256_bytes(previous_path.read_bytes()) != previous_digests.get(stem)
        ):
            raise DeployError(f"previous Worker role config changed during cutover: {stem}")
        candidate = load_json_object(previous_path, f"previous Worker role config {stem}")
        candidate.update({
            "deployment_id": activation["deploymentId"],
            "worker_release_digest": activation["workerReleaseDigest"],
            "worker_source_revision": transition["candidateSourceRevision"],
            "ownership_epoch": epoch,
        })
        path = candidate_dir / f"{stem}.json"
        expected = canonical_bytes(candidate)
        if path.exists() or path.is_symlink():
            if not path.is_file() or path.is_symlink() or path.read_bytes() != expected:
                raise DeployError(f"candidate Worker role config changed during recovery: {stem}")
        else:
            write_protected_json(path, candidate)
        candidate_paths[stem] = str(path)
        candidate_digests[stem] = sha256_bytes(path.read_bytes())
    sealed_candidate_digests = transition.get("candidateConfigDigests")
    if sealed_candidate_digests is None:
        transition = {**transition, "candidateConfigDigests": candidate_digests}
        state = replace_state(
            context,
            expected_phase="worker-image-replace-in-progress",
            updates={"workerImageTransition": transition},
        )
    elif sealed_candidate_digests != candidate_digests:
        raise DeployError("candidate Worker role config seals changed during recovery")
    if read_worker_ownership(context.envv) != expected_after:
        raise DeployError("Runtime Worker ownership does not match all candidate receipts")

    candidate_state = {**dict(state), "workerRoleConfigOverrides": candidate_paths}
    try:
        for stem in context.worker_stems:
            name = str(containers["workers"][stem])
            path = Path(candidate_paths[stem])
            ready_url = f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz"
            run_detached_container(
                context, name=name, image=runtime_worker_image,
                env_file=context.run_root / "private" / f"worker-{stem}.env",
                mounts=worker_container_mounts(context, candidate_state, stem, path),
                healthcheck_url=ready_url,
                execute=execute,
            )
            wait_ready(ready_url, name)
        if verify_catalog_after(context, state) != authority or read_worker_ownership(context.envv) != expected_after:
            raise DeployError("Catalog or Worker ownership changed during candidate startup")
        present_after_start = set(owned_containers())
        if present_after_start != set(expected_order):
            raise DeployError("Runtime Worker candidate startup left an incomplete serving inventory")
        for name in worker_names:
            if not verify_container_identity(
                context, name, runtime_worker_image, capture_output=capture_output,
            ):
                raise DeployError(f"Runtime Worker candidate stopped after readiness: {name}")
    except (DeployError, OSError, subprocess.CalledProcessError) as exc:
        present_after_failure = set(owned_containers())
        diagnostics: list[dict[str, str]] = []
        for name in sorted(worker_names & present_after_failure):
            entry = {"container": name}
            for label, command in (
                ("state", ("docker", "inspect", "--format", "{{json .State}}", name)),
                ("logs", ("docker", "logs", "--tail", "200", name)),
            ):
                try:
                    value = capture_diagnostic(command)
                    entry[label] = value[-65536:]
                except (DeployError, OSError, subprocess.CalledProcessError) as diagnostic_error:
                    entry[label + "Error"] = str(diagnostic_error)
            diagnostics.append(entry)
        print(json.dumps({
            "schema": "nexuscrm.serviceification-l0-worker-image-failure/v1",
            "runId": context.run_id,
            "operationId": transition["operationId"],
            "failure": str(exc),
            "containers": diagnostics,
        }, sort_keys=True, separators=(",", ":")), file=sys.stderr)
        for name in worker_names & present_after_failure:
            execute(("docker", "rm", "-f", name))
        raise DeployError(
            f"Runtime Worker candidate remains stopped for forward recovery: {exc}"
        ) from exc

    operations = state.get("workerImageOperations", [])
    if not isinstance(operations, list):
        raise DeployError("Worker image replacement evidence has an unknown shape")
    replace_state(
        context, expected_phase="worker-image-replace-in-progress", updates={
            "phase": "serving",
            "workerImageTransition": None,
            "workerImageOverride": runtime_worker_image,
            "workerImageSourceRevisionOverride": transition["candidateSourceRevision"],
            "workerRoleConfigOverrides": candidate_paths,
            "workerRoleConfigDigests": candidate_digests,
            "workerOwnershipAfter": worker_ownership_payload(expected_after),
            "workerImageOperations": [*operations[-63:], {
                **transition,
                "activationOutputDigests": {
                    stem: sha256_bytes((output_dir / f"{stem}.json").read_bytes())
                    for stem in context.worker_stems
                },
                "candidateConfigDigests": candidate_digests,
                "result": "activated",
            }],
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "worker-image-activated",
        "runId": context.run_id,
        "image": runtime_worker_image,
        "sourceRevision": transition["candidateSourceRevision"],
        "workerRoleCount": len(context.worker_stems),
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
    }


def worker_runtime_config_candidate(
    current: Mapping[str, Any], stem: str, *,
    parallelism: int | None = None,
    poll_interval: str | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
    """Build one bounded L0 runtime tuning update without changing release identity."""
    if (parallelism is None) == (poll_interval is None):
        raise DeployError("exactly one Worker runtime setting must be provided")
    expected_family = f"runtime.{stem.replace('-', '-')}.v1"
    if (
        current.get("schema") != "nexuscrm.runtime-worker-role/v3"
        or current.get("work_family") != expected_family
    ):
        raise DeployError(f"active {stem} config is not mutable L0 v3 authority")
    candidate = copy.deepcopy(dict(current))
    if parallelism is not None:
        if stem not in L0_PARALLELISM_MUTABLE_WORKER_ROLES:
            raise DeployError("only the record-event-dispatch role has mutable parallelism")
        if (
            not isinstance(parallelism, int) or isinstance(parallelism, bool)
            or not 1 <= parallelism <= 15
            or not isinstance(current.get("parallelism"), int)
        ):
            raise DeployError("record-event-dispatch parallelism must be between 1 and 15")
        patch = {
            "parallelism": parallelism,
            "db_max_open_conns": 2 * parallelism + 1,
            "db_max_idle_conns": parallelism,
        }
    else:
        if stem not in L0_POLL_INTERVAL_MUTABLE_WORKER_ROLES:
            raise DeployError("only the record-command-lifecycle role has mutable poll interval")
        poll_interval = validate_record_command_poll_interval(poll_interval)
        if not isinstance(current.get("poll_interval"), str):
            raise DeployError("active record-command-lifecycle config has no poll interval")
        patch = {"poll_interval": poll_interval}
    candidate.update(patch)
    return candidate, patch


def validate_record_command_poll_interval(poll_interval: str | None) -> str:
    match = re.fullmatch(r"([1-9][0-9]?)s", str(poll_interval))
    if match is None or int(match.group(1)) > 60:
        raise DeployError("record-command-lifecycle poll interval must be 1s to 60s")
    return str(poll_interval)


def remove_exact_worker_container(
    context: D3Context, name: str, image: str, *,
    execute: Callable[[Sequence[str]], None],
    capture_output: Callable[[Sequence[str]], str],
) -> None:
    if name not in set(owned_containers()):
        return
    running = verify_container_identity(
        context, name, image, capture_output=capture_output,
    )
    if running:
        execute(("docker", "stop", "--time", "30", name))
    execute(("docker", "rm", name))


def reconfigure_worker_role(
    context: D3Context, stem: str, *,
    parallelism: int | None = None,
    poll_interval: str | None = None,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
) -> dict[str, Any]:
    """Atomically update one L0 Worker runtime config and restart only that role."""
    if poll_interval is not None:
        poll_interval = validate_record_command_poll_interval(poll_interval)
    state = require_state(
        context, expected_phase={"serving", "worker-runtime-config-in-progress"},
    )
    transition = state.get("runtimeWorkerConfigTransition")
    if state["phase"] == "serving":
        authority = verify_catalog_after(context, state)
        ownership = require_worker_ownership_after(state)
        if read_worker_ownership(context.envv) != ownership:
            raise DeployError("Runtime Worker ownership changed before L0 reconfiguration")
        role_digests = verify_worker_role_config_seals(state)
        current_path = worker_role_config_path(state, stem)
        current_config = load_json_object(current_path, f"active Worker role config {stem}")
        candidate, config_patch = worker_runtime_config_candidate(
            current_config, stem,
            parallelism=parallelism,
            poll_interval=poll_interval,
        )
        candidate_bytes = canonical_bytes(candidate)
        candidate_digest = sha256_bytes(candidate_bytes)
        if candidate_digest == role_digests[stem]:
            return {
                "schema": EVIDENCE_SCHEMA,
                "stage": "worker-runtime-config-unchanged",
                "runId": context.run_id,
                "role": stem,
                "configDigest": candidate_digest,
                "configPatch": config_patch,
            }
        operation_id = new_uuid7()
        operation_root = context.run_root / "l0-runtime-config" / operation_id
        previous_path = operation_root / "previous.json"
        candidate_path = operation_root / "candidate.json"
        if operation_root.exists() or operation_root.is_symlink():
            raise DeployError("L0 Worker runtime config operation evidence already exists")
        operation_root.mkdir(parents=True, mode=0o700)
        write_protected_json(previous_path, current_config)
        write_protected_json(candidate_path, candidate)
        ownership_digest = sha256_bytes(canonical_bytes(worker_ownership_payload(ownership)))
        transition = {
            "schema": L0_WORKER_RUNTIME_CONFIG_SCHEMA,
            "operationId": operation_id,
            "role": stem,
            "configPatch": config_patch,
            "previousConfig": str(previous_path),
            "previousDigest": role_digests[stem],
            "candidateConfig": str(candidate_path),
            "candidateDigest": candidate_digest,
            "runtimeWorkerImage": effective_worker_image(context, state),
            "catalogRevision": authority.catalog_revision,
            "securityEpoch": authority.security_epoch,
            "workerOwnershipDigest": ownership_digest,
        }
        state = replace_state(
            context, expected_phase="serving", updates={
                "phase": "worker-runtime-config-in-progress",
                "runtimeWorkerConfigTransition": transition,
            },
        )
    if not isinstance(transition, dict) or transition.get("schema") != L0_WORKER_RUNTIME_CONFIG_SCHEMA:
        raise DeployError("L0 Worker runtime config transition has an unknown shape")
    requested_patch = {
        "parallelism": parallelism,
        "db_max_open_conns": 2 * parallelism + 1,
        "db_max_idle_conns": parallelism,
    } if parallelism is not None else {
        "poll_interval": poll_interval,
    }
    transition_patch = transition.get("configPatch")
    legacy_parallelism_matches = (
        transition_patch is None
        and poll_interval is None
        and transition.get("parallelism") == parallelism
    )
    if (
        transition.get("role") != stem
        or (transition_patch != requested_patch and not legacy_parallelism_matches)
    ):
        raise DeployError("another L0 Worker runtime config transition is already active")
    candidate_path = Path(str(transition.get("candidateConfig", "")))
    previous_path = Path(str(transition.get("previousConfig", "")))
    for path, digest, label in (
        (candidate_path, transition.get("candidateDigest"), "candidate"),
        (previous_path, transition.get("previousDigest"), "previous"),
    ):
        if (
            not path.is_file() or path.is_symlink()
            or sha256_bytes(path.read_bytes()) != digest
        ):
            raise DeployError(f"sealed L0 Worker {label} config changed")
    authority = verify_catalog_after(context, state)
    ownership = require_worker_ownership_after(state)
    ownership_digest = sha256_bytes(canonical_bytes(worker_ownership_payload(ownership)))
    if (
        authority.catalog_revision != transition.get("catalogRevision")
        or authority.security_epoch != transition.get("securityEpoch")
        or ownership_digest != transition.get("workerOwnershipDigest")
        or read_worker_ownership(context.envv) != ownership
    ):
        raise DeployError("Catalog or Worker ownership changed during L0 reconfiguration")
    expected_order = expected_serving_container_order(state)
    if set(owned_containers()) != set(expected_order):
        raise DeployError("L0 reconfiguration requires the complete serving container inventory")
    name = str(state["containers"]["workers"][stem])
    image, rebound_from = resolve_worker_runtime_config_image(context, state, transition)
    if not verify_container_identity(context, name, image, capture_output=capture_output):
        raise DeployError("L0 reconfiguration requires the running exact Worker image")
    if rebound_from is not None:
        transition = {
            **transition,
            "runtimeWorkerImage": image,
            "runtimeWorkerImageReboundFrom": rebound_from,
        }
        state = replace_state(
            context, expected_phase="worker-runtime-config-in-progress", updates={
                "runtimeWorkerConfigTransition": transition,
            },
        )
    ready_url = f"http://127.0.0.1:{context.ports.worker_admin[stem]}/readyz"
    try:
        remove_exact_worker_container(
            context, name, image, execute=execute, capture_output=capture_output,
        )
        run_detached_container(
            context, name=name, image=image,
            env_file=context.run_root / "private" / f"worker-{stem}.env",
            mounts=worker_container_mounts(context, state, stem, candidate_path),
            healthcheck_url=ready_url,
            execute=execute,
        )
        wait_ready(ready_url, name)
        authority_after = verify_catalog_after(context, state)
        ownership_after = read_worker_ownership(context.envv)
        if authority_after != authority or ownership_after != ownership:
            raise DeployError("L0 reconfiguration changed Catalog or Worker ownership")
    except (DeployError, OSError, subprocess.CalledProcessError) as exc:
        remove_exact_worker_container(
            context, name, image, execute=execute, capture_output=capture_output,
        )
        run_detached_container(
            context, name=name, image=image,
            env_file=context.run_root / "private" / f"worker-{stem}.env",
            mounts=worker_container_mounts(context, state, stem, previous_path),
            healthcheck_url=ready_url,
            execute=execute,
        )
        wait_ready(ready_url, name)
        operations = state.get("runtimeWorkerConfigOperations", [])
        if not isinstance(operations, list):
            raise DeployError("L0 Worker runtime config evidence has an unknown shape") from exc
        rollback_receipt = {**transition, "result": "rolled-back"}
        replace_state(
            context, expected_phase="worker-runtime-config-in-progress", updates={
                "phase": "serving",
                "runtimeWorkerConfigTransition": None,
                "runtimeWorkerConfigOperations": [*operations[-63:], rollback_receipt],
            },
        )
        raise DeployError(f"L0 Worker runtime config update rolled back: {exc}") from exc

    role_digests = dict(state["workerRoleConfigDigests"])
    role_digests[stem] = str(transition["candidateDigest"])
    overrides = state.get("workerRoleConfigOverrides", {})
    if not isinstance(overrides, dict):
        raise DeployError("L0 Worker runtime override map has an unknown shape")
    operations = state.get("runtimeWorkerConfigOperations", [])
    if not isinstance(operations, list):
        raise DeployError("L0 Worker runtime config evidence has an unknown shape")
    receipt = {**transition, "result": "activated"}
    replace_state(
        context, expected_phase="worker-runtime-config-in-progress", updates={
            "phase": "serving",
            "runtimeWorkerConfigTransition": None,
            "workerRoleConfigDigests": role_digests,
            "workerRoleConfigOverrides": {**overrides, stem: str(candidate_path)},
            "runtimeWorkerConfigOperations": [*operations[-63:], receipt],
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "worker-runtime-config-activated",
        "runId": context.run_id,
        "role": stem,
        "container": name,
        "image": image,
        "previousConfigDigest": transition["previousDigest"],
        "configDigest": transition["candidateDigest"],
        "configPatch": requested_patch,
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
    }


def replace_frontend_image(
    context: D3Context,
    *,
    frontend_image: str,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    verify_served: Callable[[D3Context, str], Mapping[str, str]] = (
        verify_served_frontend_version
    ),
    activate_nginx: Callable[..., None] = activate_nginx_conf,
) -> dict[str, Any]:
    """Replace only the immutable frontend owner; Catalog and services stay live."""
    state = require_state(
        context, expected_phase={"serving", "frontend-image-replace-in-progress"},
    )
    repository = context.images["frontend"].partition("@sha256:")[0]
    if not isinstance(frontend_image, str) or not re.fullmatch(
        re.escape(repository) + r"@sha256:[0-9a-f]{64}", frontend_image,
    ):
        raise DeployError("frontend replacement must use the owning repository and exact digest")
    image_digest = "sha256:" + frontend_image.rsplit("@sha256:", 1)[1]
    live_frontend = runtime_paths(context.env_root)["frontend_dist"]
    transition = state.get("frontendImageTransition")
    if state["phase"] == "serving":
        previous_value = state.get("publishedFrontendTarget")
        if not isinstance(previous_value, str) or not previous_value:
            raise DeployError("frontend replacement requires a published frontend owner")
        previous_target = Path(previous_value).resolve()
        if resolve_frontend_dist_target(live_frontend) != previous_target:
            raise DeployError("frontend replacement requires the exact active frontend owner")
        if state.get("frontendImageOverride") == frontend_image:
            expected_version = load_json_object(
                previous_target / "version.json", "published frontend version",
            ).get("version")
            if not isinstance(expected_version, str) or not SHA256_RE.fullmatch(expected_version):
                raise DeployError("published frontend owner has no exact version")
            try:
                served = dict(verify_served(context, expected_version))
            except DeployError:
                # State equality is not serving equality.  Fall through and repair
                # the nginx content owner from the immutable image.
                pass
            else:
                return {
                    "schema": EVIDENCE_SCHEMA,
                    "stage": "frontend-image-unchanged",
                    "runId": context.run_id,
                    "image": frontend_image,
                    "servedVersion": expected_version,
                    "served": served,
                }
        authority = verify_catalog_after(context, state)
        ownership = require_worker_ownership_after(state)
        if read_worker_ownership(context.envv) != ownership:
            raise DeployError("Runtime Worker ownership changed before frontend replacement")
        nginx_content = TEST2_NGINX_CONF.read_bytes()
        nginx_digest = sha256_bytes(nginx_content)
        if nginx_digest != state.get("publishedNginxConfigDigest"):
            raise DeployError("frontend replacement requires the exact active nginx owner")
        execute(("docker", "pull", "--quiet", frontend_image))
        extraction = context.run_root / "frontend-image-replacement"
        if extraction.is_symlink():
            raise DeployError("frontend replacement extraction path became a symlink")
        if extraction.exists():
            shutil.rmtree(extraction)
        extraction.mkdir(mode=0o700, parents=True)
        frontend_source = extraction / "frontend"
        source_tuple = extraction / "source-tuple.json"
        copy_from_exact_image(
            context, image=frontend_image, role="frontend-replacement",
            source="/usr/share/nexuscrm/frontend/.", destination=frontend_source,
            execute=execute,
        )
        copy_from_exact_image(
            context, image=frontend_image, role="frontend-release-authority",
            source="/usr/share/nexuscrm/release/source-tuple.json", destination=source_tuple,
            execute=execute,
        )
        verify_frontend_dist(frontend_source)
        if source_tuple.is_symlink() or not source_tuple.is_file():
            raise DeployError("frontend replacement source tuple is unavailable")
        source_tuple_digest = sha256_bytes(source_tuple.read_bytes())
        embedded_version = load_json_object(
            frontend_source / "version.json", "frontend replacement version",
        ).get("version")
        if embedded_version != source_tuple_digest:
            raise DeployError("frontend replacement content and source tuple disagree")
        revision = capture_output((
            "docker", "image", "inspect", "--format",
            '{{index .Config.Labels "org.opencontainers.image.revision"}}', frontend_image,
        ))
        binding = capture_output((
            "docker", "image", "inspect", "--format",
            '{{index .Config.Labels "io.nexuscrm.frontend.source-tuple-sha256"}}', frontend_image,
        ))
        if not re.fullmatch(r"[0-9a-f]{40}", revision) or binding != source_tuple_digest:
            raise DeployError("frontend replacement provenance labels disagree with its content")
        operation_id = new_uuid7()
        nginx_snapshot = (
            context.run_root / "frontend-image-transitions" / operation_id
            / "nginx-before.conf"
        )
        nginx_snapshot.parent.mkdir(mode=0o700, parents=True, exist_ok=False)
        nginx_snapshot.write_bytes(nginx_content)
        os.chmod(nginx_snapshot, 0o400)
        transition = {
            "schema": "nexuscrm.serviceification-d3-frontend-image-transition/v1",
            "operationId": operation_id,
            "previousTarget": str(previous_target),
            "previousDigest": state.get("publishedFrontendDigest"),
            "previousNginxConfig": str(nginx_snapshot),
            "previousNginxConfigDigest": nginx_digest,
            "previousImage": state.get("frontendImageOverride", context.images["frontend"]),
            "candidateImage": frontend_image,
            "candidateImageDigest": image_digest,
            "candidateSource": str(frontend_source),
            "sourceTupleDigest": source_tuple_digest,
            "sourceRevision": revision,
            "catalogRevision": authority.catalog_revision,
            "securityEpoch": authority.security_epoch,
            "workerOwnershipDigest": sha256_bytes(canonical_bytes(
                worker_ownership_payload(ownership),
            )),
        }
        state = replace_state(
            context, expected_phase="serving", updates={
                "phase": "frontend-image-replace-in-progress",
                "frontendImageTransition": transition,
            },
        )
    if (
        not isinstance(transition, dict)
        or transition.get("schema") != "nexuscrm.serviceification-d3-frontend-image-transition/v1"
        or transition.get("candidateImage") != frontend_image
    ):
        raise DeployError("another frontend image replacement is already active")
    authority = verify_catalog_after(context, state)
    ownership = require_worker_ownership_after(state)
    if (
        authority.catalog_revision != transition.get("catalogRevision")
        or authority.security_epoch != transition.get("securityEpoch")
        or read_worker_ownership(context.envv) != ownership
        or sha256_bytes(canonical_bytes(worker_ownership_payload(ownership)))
        != transition.get("workerOwnershipDigest")
    ):
        raise DeployError("Catalog or Worker ownership changed during frontend replacement")
    previous_target = Path(str(transition["previousTarget"])).resolve()
    previous_nginx = Path(str(transition["previousNginxConfig"]))
    previous_nginx_digest = str(transition["previousNginxConfigDigest"])
    if (
        previous_nginx.is_symlink()
        or not previous_nginx.is_file()
        or sha256_bytes(previous_nginx.read_bytes()) != previous_nginx_digest
    ):
        raise DeployError("frontend replacement lost its previous nginx owner")
    nginx_activation_attempted = False
    try:
        candidate_target, content_digest = publish_readable_frontend_release(
            context,
            Path(str(transition["candidateSource"])),
            release_digest=str(transition["candidateImageDigest"]),
        )
        if resolve_frontend_dist_target(live_frontend) != candidate_target:
            raise DeployError("frontend replacement did not publish the candidate owner")
        current_nginx = TEST2_NGINX_CONF.read_bytes()
        if sha256_bytes(current_nginx) != previous_nginx_digest:
            raise DeployError("nginx owner changed during frontend replacement")
        rendered_nginx = replace_nginx_server_block(
            remove_d3_nginx_markers(current_nginx.decode()),
            context.envv["SERVER_NAME"].split()[0],
            render_d3_nginx_server(context, candidate_target),
        )
        rendered_nginx_digest = sha256_bytes(rendered_nginx.encode())
        nginx_activation_attempted = True
        activate_nginx(
            TEST2_NGINX_CONF,
            rendered_nginx,
            healthcheck_url=(
                f"http://127.0.0.1:{context.envv.get('NGINX_LISTEN_PORT', '80')}/health"
            ),
            healthcheck_headers={"Host": context.envv["SERVER_NAME"].split()[0]},
        )
        if sha256_bytes(TEST2_NGINX_CONF.read_bytes()) != rendered_nginx_digest:
            raise DeployError("frontend replacement did not activate the exact nginx owner")
        served = dict(verify_served(context, str(transition["sourceTupleDigest"])))
        if verify_catalog_after(context, state) != authority or read_worker_ownership(context.envv) != ownership:
            raise DeployError("frontend replacement changed Catalog or Worker ownership")
    except (DeployError, OSError, subprocess.CalledProcessError) as exc:
        publish_symlink(live_frontend, previous_target)
        rollback_error: Exception | None = None
        if nginx_activation_attempted:
            try:
                activate_nginx(
                    TEST2_NGINX_CONF,
                    previous_nginx.read_text(),
                    healthcheck_url=(
                        f"http://127.0.0.1:"
                        f"{context.envv.get('NGINX_LISTEN_PORT', '80')}/health"
                    ),
                    healthcheck_headers={"Host": context.envv["SERVER_NAME"].split()[0]},
                )
            except (OSError, subprocess.CalledProcessError, DeployError) as rollback_exc:
                rollback_error = rollback_exc
        operations = state.get("frontendImageOperations", [])
        if not isinstance(operations, list):
            raise DeployError("frontend replacement evidence has an unknown shape") from exc
        replace_state(
            context, expected_phase="frontend-image-replace-in-progress", updates={
                "phase": "serving",
                "frontendImageTransition": None,
                "frontendImageOperations": [
                    *operations[-63:], {
                        **transition,
                        "result": "rollback-failed" if rollback_error else "rolled-back",
                        "failure": str(exc),
                        "rollbackFailure": str(rollback_error) if rollback_error else None,
                    },
                ],
            },
        )
        if rollback_error is not None:
            raise DeployError(
                "frontend image replacement and nginx rollback failed: "
                f"activation={exc}; rollback={rollback_error}"
            ) from rollback_error
        raise DeployError(f"frontend image replacement rolled back: {exc}") from exc
    operations = state.get("frontendImageOperations", [])
    if not isinstance(operations, list):
        raise DeployError("frontend replacement evidence has an unknown shape")
    replace_state(
        context, expected_phase="frontend-image-replace-in-progress", updates={
            "phase": "serving",
            "frontendImageTransition": None,
            "frontendImageOverride": frontend_image,
            "publishedFrontendTarget": str(candidate_target),
            "publishedFrontendDigest": content_digest,
            "publishedNginxConfigDigest": rendered_nginx_digest,
            "deploymentEvidence": None,
            "frontendImageOperations": [
                *operations[-63:], {
                    **transition,
                    "candidateTarget": str(candidate_target),
                    "contentDigest": content_digest,
                    "result": "activated",
                },
            ],
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "frontend-image-activated",
        "runId": context.run_id,
        "image": frontend_image,
        "contentDigest": content_digest,
        "sourceTupleDigest": transition["sourceTupleDigest"],
        "served": served,
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
    }


def replace_serving_images(
    context: D3Context,
    *,
    runtime_api_image: str | None,
    integration_service_image: str | None,
    sfcrm_app_service_image: str | None,
    runtime_api_profiling_enabled: bool | None = None,
    runtime_api_sql_diagnostics_enabled: bool | None = None,
    runtime_api_db_pool_size: int | None = None,
    failure_evidence: Path | None = None,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    capture_diagnostic: Callable[[Sequence[str]], str] = default_capture_combined,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
) -> dict[str, Any]:
    """Replace serving implementations without changing Catalog or release content."""
    state = require_state(
        context, expected_phase={"serving", "service-image-replace-in-progress"},
    )
    service_roles = ("runtime_api", "integration_service", "sfcrm_app_service")
    requested = {
        role: image for role, image in {
            "runtime_api": runtime_api_image,
            "integration_service": integration_service_image,
            "sfcrm_app_service": sfcrm_app_service_image,
        }.items() if image
    }
    if not requested:
        raise DeployError("service image replacement requires at least one exact image")
    current = effective_service_images(context, state)
    for role, image in requested.items():
        repository = context.images[role].partition("@sha256:")[0]
        if not isinstance(image, str) or not re.fullmatch(
            re.escape(repository) + r"@sha256:[0-9a-f]{64}", image,
        ):
            raise DeployError(f"{role} replacement must use the owning repository and exact digest")
    candidate = {**current, **requested}
    artifacts = state.get("servingArtifacts")
    if not isinstance(artifacts, dict) or not isinstance(artifacts.get("runtimeEnv"), str):
        raise DeployError("service replacement lost Runtime environment authority")
    runtime_env_path = Path(str(artifacts["runtimeEnv"]))
    current_runtime_env = load_env_file(runtime_env_path)
    if runtime_api_db_pool_size is not None and not 2 <= runtime_api_db_pool_size <= 256:
        raise DeployError("Runtime API database pool size must be between 2 and 256")
    previous_db_max_open = current_runtime_env.get("NEXUSCRM_DB_MAX_OPEN_CONNS")
    previous_db_max_idle = current_runtime_env.get("NEXUSCRM_DB_MAX_IDLE_CONNS")
    candidate_db_max_open = (
        str(runtime_api_db_pool_size)
        if runtime_api_db_pool_size is not None else previous_db_max_open
    )
    candidate_db_max_idle = (
        str(runtime_api_db_pool_size)
        if runtime_api_db_pool_size is not None else previous_db_max_idle
    )
    db_pool_changed = (
        runtime_api_db_pool_size is not None
        and (
            candidate_db_max_open != previous_db_max_open
            or candidate_db_max_idle != previous_db_max_idle
        )
    )
    previous_profiling_value = current_runtime_env.get(
        "NEXUSCRM_RUNTIME_API_PROFILING_ENABLED",
    )
    current_profiling_enabled = previous_profiling_value == "true"
    candidate_profiling_value = (
        "true" if runtime_api_profiling_enabled else "false"
    ) if runtime_api_profiling_enabled is not None else previous_profiling_value
    profiling_changed = (
        runtime_api_profiling_enabled is not None
        and runtime_api_profiling_enabled != current_profiling_enabled
    )
    previous_sql_diagnostics_value = current_runtime_env.get(
        "NEXUSCRM_RUNTIME_API_SQL_DIAGNOSTICS_ENABLED",
    )
    current_sql_diagnostics_enabled = previous_sql_diagnostics_value == "true"
    candidate_sql_diagnostics_value = (
        "true" if runtime_api_sql_diagnostics_enabled else "false"
    ) if runtime_api_sql_diagnostics_enabled is not None else previous_sql_diagnostics_value
    sql_diagnostics_changed = (
        runtime_api_sql_diagnostics_enabled is not None
        and runtime_api_sql_diagnostics_enabled != current_sql_diagnostics_enabled
    )
    transition = state.get("serviceImageTransition")
    if state["phase"] == "serving":
        if (
            all(candidate[role] == current[role] for role in requested)
            and not profiling_changed
            and not sql_diagnostics_changed
            and not db_pool_changed
        ):
            return {
                "schema": EVIDENCE_SCHEMA,
                "stage": "service-images-unchanged",
                "runId": context.run_id,
                "images": requested,
            }
        authority = verify_catalog_after(context, state)
        ownership = require_worker_ownership_after(state)
        if read_worker_ownership(context.envv) != ownership:
            raise DeployError("Runtime Worker ownership changed before service replacement")
        expected = container_images_by_name(context, state)
        if set(owned_containers()) != set(expected_serving_container_order(state)):
            raise DeployError("service replacement requires the complete serving container inventory")
        for name, image in expected.items():
            if not verify_container_identity(context, name, image, capture_output=capture_output):
                raise DeployError(f"service replacement requires a running exact container: {name}")
        for role in requested:
            execute(("docker", "pull", "--quiet", candidate[role]))
        transition = {
            "schema": L0_SERVICE_IMAGE_TRANSITION_SCHEMA,
            "operationId": new_uuid7(),
            "previousImages": {role: current[role] for role in service_roles},
            "candidateImages": {role: candidate[role] for role in service_roles},
            "requestedRoles": sorted(requested),
            "previousRuntimeProfilingValue": previous_profiling_value,
            "candidateRuntimeProfilingValue": candidate_profiling_value,
            "previousRuntimeSQLDiagnosticsValue": previous_sql_diagnostics_value,
            "candidateRuntimeSQLDiagnosticsValue": candidate_sql_diagnostics_value,
            "previousRuntimeDBMaxOpenConns": previous_db_max_open,
            "previousRuntimeDBMaxIdleConns": previous_db_max_idle,
            "candidateRuntimeDBMaxOpenConns": candidate_db_max_open,
            "candidateRuntimeDBMaxIdleConns": candidate_db_max_idle,
            "catalogRevision": authority.catalog_revision,
            "securityEpoch": authority.security_epoch,
            "workerOwnershipDigest": sha256_bytes(canonical_bytes(
                worker_ownership_payload(ownership),
            )),
        }
        state = replace_state(
            context, expected_phase="serving", updates={
                "phase": "service-image-replace-in-progress",
                "serviceImageTransition": transition,
            },
        )
    if (
        not isinstance(transition, dict)
        or transition.get("schema") != L0_SERVICE_IMAGE_TRANSITION_SCHEMA
        or transition.get("candidateImages") != {
            role: candidate[role] for role in service_roles
        }
        or transition.get("candidateRuntimeProfilingValue") != candidate_profiling_value
        or transition.get("candidateRuntimeSQLDiagnosticsValue") != candidate_sql_diagnostics_value
        or transition.get("candidateRuntimeDBMaxOpenConns") != candidate_db_max_open
        or transition.get("candidateRuntimeDBMaxIdleConns") != candidate_db_max_idle
    ):
        raise DeployError("another service image replacement is already active")

    authority = verify_catalog_after(context, state)
    ownership = require_worker_ownership_after(state)
    if (
        authority.catalog_revision != transition.get("catalogRevision")
        or authority.security_epoch != transition.get("securityEpoch")
        or read_worker_ownership(context.envv) != ownership
        or sha256_bytes(canonical_bytes(worker_ownership_payload(ownership)))
        != transition.get("workerOwnershipDigest")
    ):
        raise DeployError("Catalog or Worker ownership changed during service replacement")
    artifacts = state.get("servingArtifacts")
    catalog_artifacts = state.get("catalogArtifacts")
    containers = state.get("containers")
    if not isinstance(artifacts, dict) or not isinstance(catalog_artifacts, dict) or not isinstance(containers, dict):
        raise DeployError("service replacement lost serving artifacts")
    runtime_name = str(containers["runtime_api"])
    integration_name = str(containers["integration_service"])
    sfcrm_name = str(containers["sfcrm_app_service"])
    previous = dict(transition["previousImages"])
    next_images = dict(transition["candidateImages"])
    storage_mounts = preflight_runtime_filesystem_storage_mounts(
        context,
        runtime_image=next_images["runtime_api"],
        environment_paths=(runtime_env_path,),
        execute=execute,
    )
    replace_runtime = "runtime_api" in transition["requestedRoles"]
    replace_integration = "integration_service" in transition["requestedRoles"]
    replace_sfcrm = "sfcrm_app_service" in transition["requestedRoles"]
    # Runtime and sfcrm share a private serving trust path, so preserve their
    # existing paired restart. Integration has its own token/trust artifacts
    # and can be cut over independently.
    targets = [
        *( [sfcrm_name] if replace_runtime or replace_sfcrm else []),
        *( [runtime_name] if replace_runtime else []),
        *( [integration_name] if replace_integration else []),
    ]

    def write_runtime_settings(
        profiling: str | None,
        sql_diagnostics: str | None,
        db_max_open: str | None,
        db_max_idle: str | None,
    ) -> None:
        values = load_env_file(runtime_env_path)
        if profiling is None:
            values.pop("NEXUSCRM_RUNTIME_API_PROFILING_ENABLED", None)
        else:
            values["NEXUSCRM_RUNTIME_API_PROFILING_ENABLED"] = profiling
        if sql_diagnostics is None:
            values.pop("NEXUSCRM_RUNTIME_API_SQL_DIAGNOSTICS_ENABLED", None)
        else:
            values["NEXUSCRM_RUNTIME_API_SQL_DIAGNOSTICS_ENABLED"] = sql_diagnostics
        for key, value in (
            ("NEXUSCRM_DB_MAX_OPEN_CONNS", db_max_open),
            ("NEXUSCRM_DB_MAX_IDLE_CONNS", db_max_idle),
        ):
            if value is None:
                values.pop(key, None)
            else:
                values[key] = value
        write_env_file(runtime_env_path, values)

    def remove_target(name: str) -> None:
        if name not in set(owned_containers()):
            return
        fields = capture_output((
            "docker", "inspect", "--format",
            '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}',
            name,
        )).split("\t")
        if len(fields) != 2 or fields[1] != context.run_id or fields[0] not in set(previous.values()) | set(next_images.values()):
            raise DeployError(f"service replacement found an unowned container: {name}")
        execute(("docker", "rm", "-f", name))

    def start_pair(images: Mapping[str, str]) -> None:
        present = set(owned_containers())
        if replace_runtime:
            ensure_detached_container(
                context, name=runtime_name, image=images["runtime_api"],
                env_file=Path(str(artifacts["runtimeEnv"])),
                mounts=(
                    (Path(str(artifacts["runtimeConfig"])), "/etc/nexuscrm/serving.json", True),
                    (Path(str(artifacts["runtimeWorkloadIdentities"])), "/run/secrets/workload-identities.json", True),
                    (Path(str(artifacts["runtimeSigningKeys"])), "/run/secrets/signing-keys.json", True),
                    (Path(str(artifacts["runtimeCertificate"])), "/run/tls/server.crt", True),
                    (Path(str(artifacts["runtimePrivateKey"])), "/run/tls/server.key", True),
                    (Path(str(catalog_artifacts["store"])), "/var/lib/nexuscrm/catalog", True),
                    *storage_mounts,
                ),
                ready_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
                healthcheck_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
                present=present, execute=execute, capture_output=capture_output, wait_ready=wait_ready,
            )
        if replace_runtime or replace_sfcrm:
            ensure_detached_container(
                context, name=sfcrm_name, image=images["sfcrm_app_service"],
                env_file=Path(str(artifacts["sfcrmEnv"])),
                mounts=(
                    (Path(str(artifacts["sfcrmToken"])), "/run/secrets/workload.token", True),
                    (Path(str(artifacts["sfcrmTrust"])), "/run/trust/runtime-ca.crt", True),
                ),
                ready_url=f"http://127.0.0.1:{context.ports.sfcrm_http}/readyz",
                healthcheck_url=None,
                present=present, execute=execute, capture_output=capture_output, wait_ready=wait_ready,
            )
        if replace_integration:
            ensure_detached_container(
                context, name=integration_name, image=images["integration_service"],
                env_file=Path(str(artifacts["integrationEnv"])),
                mounts=(
                    (Path(str(artifacts["integrationToken"])), "/run/secrets/workload.token", True),
                    (Path(str(artifacts["integrationTrust"])), "/run/trust/runtime-ca.crt", True),
                ),
                ready_url=f"http://127.0.0.1:{context.ports.integration_http}/readyz",
                healthcheck_url=None,
                present=present, execute=execute, capture_output=capture_output, wait_ready=wait_ready,
            )

    def capture_candidate_failure(exc: BaseException) -> Path | None:
        if failure_evidence is None:
            return None
        containers_evidence: list[dict[str, Any]] = []
        for name in targets:
            entry: dict[str, Any] = {"container": name}
            for label, command in (
                (
                    "state",
                    ("docker", "inspect", "--format", "{{json .State}}", name),
                ),
                ("logs", ("docker", "logs", "--tail", "400", name)),
            ):
                try:
                    value = capture_diagnostic(command)
                    # Startup logs are evidence, not an unbounded archive. Keep
                    # the tail even if one application log line is unusually
                    # large.
                    if len(value) > 131072:
                        value = "[truncated to final 131072 characters]\n" + value[-131072:]
                    entry[label] = value
                except (DeployError, OSError, subprocess.CalledProcessError, KeyError) as capture_error:
                    entry[label + "Error"] = str(capture_error)
            containers_evidence.append(entry)
        write_protected_json(failure_evidence, {
            "schema": L0_SERVICE_IMAGE_FAILURE_EVIDENCE_SCHEMA,
            "runId": context.run_id,
            "operationId": transition["operationId"],
            "failure": str(exc),
            "candidateImages": transition["candidateImages"],
            "requestedRoles": transition["requestedRoles"],
            "catalogRevision": transition["catalogRevision"],
            "securityEpoch": transition["securityEpoch"],
            "containers": containers_evidence,
        })
        return failure_evidence

    try:
        for name in targets:
            remove_target(name)
        write_runtime_settings(
            transition.get("candidateRuntimeProfilingValue"),
            transition.get("candidateRuntimeSQLDiagnosticsValue"),
            transition.get("candidateRuntimeDBMaxOpenConns"),
            transition.get("candidateRuntimeDBMaxIdleConns"),
        )
        start_pair(next_images)
        if verify_catalog_after(context, state) != authority or read_worker_ownership(context.envv) != ownership:
            raise DeployError("service replacement changed Catalog or Worker ownership")
    except (DeployError, OSError, subprocess.CalledProcessError) as exc:
        diagnostic_path: Path | None = None
        diagnostic_error: str | None = None
        try:
            diagnostic_path = capture_candidate_failure(exc)
        except (DeployError, OSError, subprocess.CalledProcessError) as capture_error:
            diagnostic_error = str(capture_error)
        diagnostic_suffix = (
            f"; candidate failure evidence: {diagnostic_path}"
            if diagnostic_path is not None else
            (f"; candidate failure evidence capture failed: {diagnostic_error}"
             if diagnostic_error is not None else "")
        )
        raise DeployError(
            f"service image replacement stopped for forward retry: {exc}{diagnostic_suffix}"
        ) from exc

    overrides = state.get("serviceImageOverrides", {})
    operations = state.get("serviceImageOperations", [])
    if not isinstance(overrides, dict) or not isinstance(operations, list):
        raise DeployError("service image evidence has an unknown shape")
    activated = {**overrides, **{role: candidate[role] for role in requested}}
    replace_state(
        context, expected_phase="service-image-replace-in-progress", updates={
            "phase": "serving",
            "serviceImageTransition": None,
            "serviceImageOverrides": activated,
            "serviceImageOperations": [*operations[-63:], {**transition, "result": "activated"}],
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "service-images-activated",
        "runId": context.run_id,
        "images": {role: candidate[role] for role in requested},
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
        "runtimeProfilingEnabled": transition.get("candidateRuntimeProfilingValue") == "true",
        "runtimeSQLDiagnosticsEnabled": transition.get("candidateRuntimeSQLDiagnosticsValue") == "true",
        "runtimeDBMaxOpenConns": int(str(transition["candidateRuntimeDBMaxOpenConns"])),
        "runtimeDBMaxIdleConns": int(str(transition["candidateRuntimeDBMaxIdleConns"])),
    }


def rotate_serving_tls(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
    tls_provider: Callable[..., tuple[Path, Path]] = provision_tls_identity,
) -> dict[str, Any]:
    """Rotate the private Runtime identity without changing serving authorities."""
    state = require_state(
        context,
        expected_phase={"serving", "service-image-replace-in-progress"},
    )
    phase = str(state["phase"])
    authority = verify_catalog_after(context, state)
    ownership = require_worker_ownership_after(state)
    ownership_digest = sha256_bytes(canonical_bytes(worker_ownership_payload(ownership)))
    if read_worker_ownership(context.envv) != ownership:
        raise DeployError("Runtime Worker ownership changed before serving TLS rotation")

    expected_images = container_images_by_name(context, state)
    expected_order = expected_serving_container_order(state)
    rotation = state.get("servingTLSRotation")
    live_containers = set(owned_containers())
    required_workers = set(expected_order[1:1 + len(context.worker_stems)])
    if rotation is None:
        if live_containers != set(expected_order):
            raise DeployError("serving TLS rotation requires the complete serving inventory")
    elif (
        not required_workers.issubset(live_containers)
        or not live_containers.issubset(set(expected_order))
    ):
        raise DeployError("serving TLS rotation resume found container inventory drift")
    for name in live_containers:
        image = expected_images[name]
        fields = capture_output((
            "docker", "inspect", "--format",
            '{{.Config.Image}}\t{{index .Config.Labels "io.nexuscrm.serviceification-d3.run"}}\t'
            '{{.HostConfig.NetworkMode}}\t{{.HostConfig.RestartPolicy.Name}}',
            name,
        )).split("\t")
        if fields != [image, context.run_id, "host", "unless-stopped"]:
            raise DeployError(f"serving TLS rotation found container identity drift: {name}")

    artifacts = state.get("servingArtifacts")
    catalog_artifacts = state.get("catalogArtifacts")
    containers = state.get("containers")
    if (
        not isinstance(artifacts, dict)
        or not isinstance(catalog_artifacts, dict)
        or not isinstance(containers, dict)
    ):
        raise DeployError("serving TLS rotation lost serving artifacts")

    if rotation is None:
        operation_id = new_uuid7()
        rotation_root = context.run_root / "tls-rotations" / operation_id
        certificate, private_key = tls_provider(
            rotation_root / "runtime",
            RUNTIME_SERVER_NAME,
            execute=execute,
        )
        if not certificate.is_file() or not private_key.is_file():
            raise DeployError("serving TLS rotation did not produce a complete Runtime identity")
        sfcrm_trust = rotation_root / "sfcrm" / "runtime-ca.crt"
        integration_trust = rotation_root / "integration" / "runtime-ca.crt"
        for trust_path in (sfcrm_trust, integration_trust):
            trust_path.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
            shutil.copyfile(certificate, trust_path)
            os.chmod(trust_path, 0o400)
        candidate_artifacts = {
            **artifacts,
            "runtimeCertificate": str(certificate),
            "runtimeCertificateDigest": sha256_bytes(certificate.read_bytes()),
            "runtimePrivateKey": str(private_key),
            "runtimePrivateKeyDigest": sha256_bytes(private_key.read_bytes()),
            "sfcrmTrust": str(sfcrm_trust),
            "sfcrmTrustDigest": sha256_bytes(sfcrm_trust.read_bytes()),
            "integrationTrust": str(integration_trust),
            "integrationTrustDigest": sha256_bytes(integration_trust.read_bytes()),
        }
        rotation = {
            "schema": L0_SERVING_TLS_ROTATION_SCHEMA,
            "operationId": operation_id,
            "catalogRevision": authority.catalog_revision,
            "securityEpoch": authority.security_epoch,
            "workerOwnershipDigest": ownership_digest,
            "images": {
                name: expected_images[name]
                for name in (
                    str(containers["runtime_api"]),
                    str(containers["integration_service"]),
                    str(containers["sfcrm_app_service"]),
                    *([str(containers["mcp_worker"])] if "mcp_worker" in context.images else []),
                )
            },
            "previousArtifacts": {
                key: artifacts.get(key) for key in (
                    "runtimeCertificate", "runtimePrivateKey",
                    "sfcrmTrust", "integrationTrust",
                )
            },
            "candidateArtifacts": {
                key: candidate_artifacts[key] for key in (
                    "runtimeCertificate", "runtimeCertificateDigest",
                    "runtimePrivateKey", "runtimePrivateKeyDigest",
                    "sfcrmTrust", "sfcrmTrustDigest",
                    "integrationTrust", "integrationTrustDigest",
                )
            },
            "validityDays": D3_SERVING_TLS_VALIDITY_DAYS,
        }
        state = replace_state(
            context,
            expected_phase=phase,
            updates={
                "servingArtifacts": candidate_artifacts,
                "servingTLSRotation": rotation,
            },
        )
        artifacts = candidate_artifacts
    if (
        not isinstance(rotation, dict)
        or rotation.get("schema") != L0_SERVING_TLS_ROTATION_SCHEMA
        or rotation.get("catalogRevision") != authority.catalog_revision
        or rotation.get("securityEpoch") != authority.security_epoch
        or rotation.get("workerOwnershipDigest") != ownership_digest
        or rotation.get("images") != {
            name: expected_images[name]
            for name in (
                str(containers["runtime_api"]),
                str(containers["integration_service"]),
                str(containers["sfcrm_app_service"]),
                *([str(containers["mcp_worker"])] if "mcp_worker" in context.images else []),
            )
        }
    ):
        raise DeployError("another serving TLS rotation is already active")
    candidate_artifacts = rotation.get("candidateArtifacts")
    if not isinstance(candidate_artifacts, dict) or any(
        artifacts.get(key) != value for key, value in candidate_artifacts.items()
    ):
        raise DeployError("serving TLS rotation candidate artifacts drifted")
    for path_key, digest_key in (
        ("runtimeCertificate", "runtimeCertificateDigest"),
        ("runtimePrivateKey", "runtimePrivateKeyDigest"),
        ("sfcrmTrust", "sfcrmTrustDigest"),
        ("integrationTrust", "integrationTrustDigest"),
    ):
        path = Path(str(candidate_artifacts[path_key]))
        if not path.is_file() or path.is_symlink():
            raise DeployError(f"serving TLS rotation artifact is unavailable: {path_key}")
        if sha256_bytes(path.read_bytes()) != candidate_artifacts[digest_key]:
            raise DeployError(f"serving TLS rotation artifact digest drifted: {path_key}")

    runtime_name = str(containers["runtime_api"])
    integration_name = str(containers["integration_service"])
    sfcrm_name = str(containers["sfcrm_app_service"])
    service_images = effective_service_images(context, state)
    storage_mounts = preflight_runtime_filesystem_storage_mounts(
        context,
        runtime_image=service_images["runtime_api"],
        environment_paths=(Path(str(artifacts["runtimeEnv"])),),
        execute=execute,
    )
    for name in (*([str(containers["mcp_worker"])] if "mcp_worker" in context.images else []), sfcrm_name, integration_name, runtime_name):
        if name in set(owned_containers()):
            execute(("docker", "rm", "-f", name))

    present = set(owned_containers())
    ensure_detached_container(
        context,
        name=runtime_name,
        image=service_images["runtime_api"],
        env_file=Path(str(artifacts["runtimeEnv"])),
        mounts=(
            (Path(str(artifacts["runtimeConfig"])), "/etc/nexuscrm/serving.json", True),
            (Path(str(artifacts["runtimeWorkloadIdentities"])), "/run/secrets/workload-identities.json", True),
            (Path(str(artifacts["runtimeSigningKeys"])), "/run/secrets/signing-keys.json", True),
            (Path(str(artifacts["runtimeCertificate"])), "/run/tls/server.crt", True),
            (Path(str(artifacts["runtimePrivateKey"])), "/run/tls/server.key", True),
            (Path(str(catalog_artifacts["store"])), "/var/lib/nexuscrm/catalog", True),
            *storage_mounts,
        ),
        ready_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
        healthcheck_url=f"http://127.0.0.1:{context.ports.runtime_admin}/readyz",
        present=present,
        execute=execute,
        capture_output=capture_output,
        wait_ready=wait_ready,
    )
    ensure_detached_container(
        context,
        name=integration_name,
        image=service_images["integration_service"],
        env_file=Path(str(artifacts["integrationEnv"])),
        mounts=(
            (Path(str(artifacts["integrationToken"])), "/run/secrets/workload.token", True),
            (Path(str(artifacts["integrationTrust"])), "/run/trust/runtime-ca.crt", True),
        ),
        ready_url=f"http://127.0.0.1:{context.ports.integration_http}/readyz",
        healthcheck_url=None,
        present=present,
        execute=execute,
        capture_output=capture_output,
        wait_ready=wait_ready,
    )
    ensure_detached_container(
        context,
        name=sfcrm_name,
        image=service_images["sfcrm_app_service"],
        env_file=Path(str(artifacts["sfcrmEnv"])),
        mounts=(
            (Path(str(artifacts["sfcrmToken"])), "/run/secrets/workload.token", True),
            (Path(str(artifacts["sfcrmTrust"])), "/run/trust/runtime-ca.crt", True),
        ),
        ready_url=f"http://127.0.0.1:{context.ports.sfcrm_http}/readyz",
        healthcheck_url=None,
        present=present,
        execute=execute,
        capture_output=capture_output,
        wait_ready=wait_ready,
    )
    if "mcp_worker" in context.images:
        start_mcp_worker(context, state, present=present, execute=execute,
                         capture_output=capture_output, wait_ready=wait_ready)
    if (
        verify_catalog_after(context, state) != authority
        or read_worker_ownership(context.envv) != ownership
    ):
        raise DeployError("serving TLS rotation changed Catalog or Worker ownership")

    operations = state.get("servingTLSOperations", [])
    if not isinstance(operations, list):
        raise DeployError("serving TLS rotation evidence has an unknown shape")
    receipt = {**rotation, "result": "activated"}
    replace_state(
        context,
        expected_phase=phase,
        updates={
            "servingTLSRotation": None,
            "servingTLSOperations": [*operations[-63:], receipt],
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "serving-tls-rotated",
        "runId": context.run_id,
        "catalogRevision": authority.catalog_revision,
        "securityEpoch": authority.security_epoch,
        "images": rotation["images"],
        "validityDays": rotation["validityDays"],
    }


def restart_container_for_role(state: Mapping[str, Any], role: str) -> str:
    """Resolve an operator's role by identity, never by serving-list position."""
    containers = state["containers"]
    key = {"runtime-api": "runtime_api", "integration-service": "integration_service",
           "sfcrm-app-service": "sfcrm_app_service", "mcp-worker": "mcp_worker"}.get(role)
    if key is not None:
        name = containers.get(key)
    elif role.startswith("worker:") and role.removeprefix("worker:") in state_worker_stems(state):
        name = containers["workers"].get(role.removeprefix("worker:"))
    else:
        name = None
    if not isinstance(name, str) or not name:
        raise DeployError("restart role must name a process selected by this D3 release")
    return name


def exercise_restart(
    context: D3Context,
    role: str,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
    capture_output: Callable[[Sequence[str]], str] = default_capture,
    wait_ready: Callable[[str, str], None] = wait_http_ready,
) -> dict[str, Any]:
    state = require_state(context, expected_phase="serving")
    authority_before = verify_catalog_after(context, state)
    ownership_before = require_worker_ownership_after(state)
    if read_worker_ownership(context.envv) != ownership_before:
        raise DeployError("Runtime Worker ownership changed before the restart probe")
    targets = dict(serving_health_targets(context, state))
    name = restart_container_for_role(state, role)
    expected_image = container_images_by_name(context, state)[name]
    if not verify_container_identity(
        context, name, expected_image, capture_output=capture_output,
    ):
        raise DeployError("restart probe requires a running exact D3 container")
    execute(("docker", "restart", "--time", "30", name))
    wait_ready(targets[name], name)
    if role == "runtime-api":
        for client in ("integration_service", "sfcrm_app_service", "mcp_worker"):
            if client in state["containers"]:
                client_name = str(state["containers"][client])
                wait_ready(targets[client_name], client_name)
    authority_after = verify_catalog_after(context, state)
    ownership_after = read_worker_ownership(context.envv)
    if authority_after != authority_before or ownership_after != ownership_before:
        raise DeployError("restart changed immutable Catalog or Runtime Worker authority")
    proofs = state.get("restartProofs", [])
    if not isinstance(proofs, list):
        raise DeployError("D3 restart evidence has an unknown shape")
    proof = {
        "role": role,
        "container": name,
        "image": expected_image,
        "catalogRevision": authority_after.catalog_revision,
        "securityEpoch": authority_after.security_epoch,
        "workerOwnershipDigest": sha256_bytes(canonical_bytes(
            worker_ownership_payload(ownership_after),
        )),
    }
    replace_state(
        context, expected_phase="serving",
        updates={"restartProofs": [*proofs, proof]},
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "restart-proven",
        "runId": context.run_id,
        **proof,
    }


def require_catalog_before(state: Mapping[str, Any]) -> CatalogAuthority:
    value = state.get("catalogBefore")
    if not isinstance(value, dict):
        raise DeployError("quiesced D3 state has no Catalog rollback boundary")
    try:
        authority = CatalogAuthority(
            app_installation_id=str(value["appInstallationId"]),
            org_id=str(value["orgId"]),
            active_release_digest=str(value["activeReleaseDigest"]),
            catalog_revision=int(value["catalogRevision"]),
            runtime_audience=str(value["runtimeAudience"]),
            security_epoch=int(value["securityEpoch"]),
            active_publication_digest=str(value["activePublicationDigest"]),
            schema_generation=int(value["schemaGeneration"]),
            engine_generation=int(value["engineGeneration"]),
        )
    except (KeyError, TypeError, ValueError) as exc:
        raise DeployError("quiesced D3 Catalog rollback boundary has an unknown shape") from exc
    if (
        not authority.app_installation_id
        or not authority.org_id
        or not authority.runtime_audience
        or not SHA256_RE.fullmatch(authority.active_release_digest)
        or not SHA256_RE.fullmatch(authority.active_publication_digest)
        or min(
            authority.catalog_revision, authority.security_epoch,
            authority.schema_generation, authority.engine_generation,
        ) <= 0
    ):
        raise DeployError("quiesced D3 Catalog rollback boundary is incomplete")
    return authority


def require_catalog_start(state: Mapping[str, Any]) -> dict[str, Any]:
    value = state.get("catalogStart")
    if not isinstance(value, dict):
        raise DeployError("quiesced D3 state has no Catalog start mode")
    mode = value.get("mode")
    if mode == "UPGRADE":
        authority = require_catalog_before(state)
        return {
            "mode": mode,
            "refreshActiveRelease": bool(value.get("refreshActiveRelease", False)),
            "orgId": authority.org_id,
            "runtimeAudience": authority.runtime_audience,
            "appInstallationId": authority.app_installation_id,
            "expectedCatalogRevision": authority.catalog_revision,
            "expectedSecurityEpoch": authority.security_epoch,
            "schemaGeneration": authority.schema_generation,
            "engineGeneration": authority.engine_generation,
        }
    if set(value) != {
        "mode", "orgId", "runtimeAudience", "expectedCatalogRevision",
        "expectedSecurityEpoch", "schemaGeneration", "engineGeneration",
    }:
        raise DeployError("D3 PREPARE start authority has an unknown shape")
    if (
        mode != "PREPARE"
        or not isinstance(value["orgId"], str)
        or not value["orgId"]
        or len(value["orgId"].encode("utf-8")) > 255
        or any(character.isspace() for character in value["orgId"])
        or value["runtimeAudience"] != "nexuscrm-runtime-api"
        or not isinstance(value["expectedCatalogRevision"], int)
        or isinstance(value["expectedCatalogRevision"], bool)
        or value["expectedCatalogRevision"] < 0
        or not isinstance(value["expectedSecurityEpoch"], int)
        or isinstance(value["expectedSecurityEpoch"], bool)
        or value["expectedSecurityEpoch"] < 0
        or value["schemaGeneration"] != 1
        or value["engineGeneration"] != 1
    ):
        raise DeployError("D3 PREPARE start authority is incomplete")
    return {**value, "appInstallationId": ""}


def catalog_input_idempotency_key(
    bom: Mapping[str, Any],
    start: Mapping[str, Any],
    *,
    l1_version: str,
    l2_version: str,
    integration_bindings_digest: str,
    outbound_mail_actions_digest: str,
    approval_publication_digest: str,
    operation_attempt_digest: str = "",
) -> str:
    input_digest = sha256_bytes(canonical_bytes({
        # Catalog content is versioned by L1 + L2. The resolved BOM and image
        # hashes are immutable deployment evidence, not Catalog dependencies.
        "appId": bom["app_installation"]["app_id"],
        "l1Version": l1_version,
        "l2Version": l2_version,
        "integrationBindingsDigest": integration_bindings_digest,
        "outboundMailActionsDigest": outbound_mail_actions_digest,
        "approvalPublicationDigest": approval_publication_digest,
    })).removeprefix("sha256:")
    attempt_suffix = ""
    if operation_attempt_digest:
        if not SHA256_RE.fullmatch(operation_attempt_digest):
            raise DeployError("Catalog operation attempt digest is malformed")
        # Catalog content remains owned by L1 + L2. This suffix is separate
        # lifecycle identity for a deliberately new signed successor after the
        # prior operation was durably abandoned; it is not a content version.
        attempt_suffix = f"-a{operation_attempt_digest.removeprefix('sha256:')}"
    return (
        f"d3-test2-{input_digest}-{start['mode'].lower()}"
        f"-r{start['expectedCatalogRevision']}"
        f"{attempt_suffix}"
    )


def catalog_successor_attempt_digest(
    abandonment_evidence_digest: str,
    successor_desired_bom_digest: str,
) -> str:
    """Bind a post-abandonment lifecycle attempt to its signed successor BOM."""
    if (
        not SHA256_RE.fullmatch(abandonment_evidence_digest)
        or not SHA256_RE.fullmatch(successor_desired_bom_digest)
    ):
        raise DeployError("Catalog successor attempt identity is malformed")
    return sha256_bytes(canonical_bytes({
        "schema": "nexuscrm.test2-catalog-successor-attempt/v1",
        "abandonmentEvidenceDigest": abandonment_evidence_digest,
        "successorDesiredBomDigest": successor_desired_bom_digest,
    }))


def catalog_release_versions(
    context: D3Context,
    *,
    l2_root: Path,
) -> dict[str, str]:
    l1_version = context.bom["app_installation"].get("semantic_version")
    if not isinstance(l1_version, str) or not l1_version:
        raise DeployError("Catalog input has no L1 AppRelease version")
    metadata = context.bom["l2_metadata"]
    pack_root = (l2_root / str(metadata.get("path", ""))).resolve()
    try:
        pack_root.relative_to(l2_root.resolve())
    except ValueError as exc:
        raise DeployError("Catalog L2 metadata path escapes its exact source") from exc
    manifest_path = pack_root / "pack.json"
    manifest = load_json_object(manifest_path, "L2 final-pack manifest")
    l2_version = manifest.get("version")
    if not isinstance(l2_version, str) or not l2_version:
        raise DeployError("Catalog input has no L2 final-pack version")
    return {"l1": l1_version, "l2": l2_version}


def prepare_catalog(
    context: D3Context,
    *,
    allow_empty_integration: bool = False,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> dict[str, Any]:
    state = require_state(context, expected_phase="runtime-applied")
    start = require_catalog_start(state)
    before = require_catalog_before(state) if start["mode"] == "UPGRADE" else None
    if read_optional_catalog_authority(context.envv) != before:
        raise DeployError("Catalog authority changed after D3 quiescence")
    exact_sources = state.get("exactSources")
    prepared = state.get("prepared")
    if not isinstance(exact_sources, dict) or not isinstance(prepared, dict):
        raise DeployError("D3 prepared artifact state is incomplete")
    app_packs_root = Path(str(exact_sources.get("appPacksRoot", ""))).resolve()
    l2_root = Path(str(exact_sources.get("l2Root", ""))).resolve()
    if not app_packs_root.is_dir() or not l2_root.is_dir():
        raise DeployError("D3 exact source checkout disappeared after quiescence")
    l2_commit = str(exact_sources.get("l2Commit", ""))
    if not runtime_app_shell_receipt_matches(
        state.get("runtimeAppShell"),
        l2_root=l2_root,
        l2_commit=l2_commit,
        navigation_required=any(g["capability_id"] == "runtime.application-navigation.v1"
                                for g in context.bom["catalog_consumers"]["runtime_capability_grants"]),
    ):
        state = recover_d3_release_app_shell_before_catalog(
            context,
            l2_root,
            l2_commit,
            expected_catalog=before,
            expected_worker_ownership=require_worker_ownership_before(state),
        )

    if start.get("refreshActiveRelease"):
        if before is None:
            raise DeployError("same-release Catalog refresh has no active authority")
        start = {**start, "activeReleaseDigest": before.active_release_digest}

    catalog_root = context.run_root / "catalog"
    if catalog_root.is_symlink():
        raise DeployError("incomplete D3 Catalog path became a symlink")
    preserve_immutable_store = (
        start["mode"] == "UPGRADE"
        or int(start["expectedCatalogRevision"]) > 0
    )
    if catalog_root.exists():
        if preserve_immutable_store:
            store = catalog_root / "store"
            if store.is_symlink() or (store.exists() and not store.is_dir()):
                raise DeployError("D3 Catalog upgrade store is not a regular directory")
            for child in catalog_root.iterdir():
                if child.name == "store":
                    continue
                if child.is_symlink():
                    raise DeployError(
                        f"D3 Catalog upgrade path became a symlink: {child.name}"
                    )
                if child.is_dir():
                    shutil.rmtree(child)
                elif child.is_file():
                    child.unlink()
                else:
                    raise DeployError(
                        f"D3 Catalog upgrade root contains unsupported path: {child.name}"
                    )
        else:
            shutil.rmtree(catalog_root)
    config_dir = catalog_root / "config"
    output_dir = catalog_root / "output"
    store_dir = catalog_root / "store"
    for path in (config_dir, output_dir):
        path.mkdir(parents=True, mode=0o700, exist_ok=False)
    store_dir.mkdir(parents=True, mode=0o700, exist_ok=preserve_immutable_store)
    private_dir = context.run_root / "private"
    private_dir.mkdir(mode=0o700, exist_ok=True)
    db_env_file = private_dir / "database.env"
    write_env_file(db_env_file, database_env(context.envv))
    approval_bundle = Path(str(prepared.get("approvalPublication", ""))).resolve()
    approval_bundle_digest = str(prepared.get("approvalPublicationDigest", ""))
    if (
        approval_bundle.is_symlink()
        or not approval_bundle.is_file()
        or sha256_bytes(approval_bundle.read_bytes()) != approval_bundle_digest
    ):
        raise DeployError("sealed D3 Approval publication bundle is unavailable")
    approval_container_path = "/etc/nexuscrm/approval-publication.json"

    bindings_path = config_dir / "integration-bindings.json"
    if allow_empty_integration:
        bindings_path.write_bytes(canonical_bytes({
            "schema": D3_INTEGRATION_BINDINGS_SCHEMA,
            "integrationBindings": [],
        }))
    else:
        execute((
            "python3", str(REPOSITORY_ROOT / "scripts" / "render_serviceification_d3_integration_bindings.py"),
            "--stack-lock", str(l2_root / "stack.lock.json"),
            "--runtime-env", str(context.env_file),
            "--app-packs-root", str(app_packs_root),
            "--l2-root", str(l2_root),
            "--output", str(bindings_path),
        ))
    if not bindings_path.is_file() or bindings_path.is_symlink():
        raise DeployError("D3 IntegrationBinding renderer emitted no regular authority input")

    catalog_config = config_dir / "catalog-reconcile.json"
    frontend_receipt = output_dir / "frontend-compatibility-receipt.json"
    bindings_digest = sha256_bytes(bindings_path.read_bytes())
    mail_actions_candidate = (
        l2_root
        / str(context.bom["l2_metadata"]["path"])
        / "actions"
        / "invocable_standard_actions.json"
    )
    if mail_actions_candidate.is_symlink():
        raise DeployError("Catalog outbound-mail action authority must not be a symlink")
    mail_actions_path = mail_actions_candidate.resolve()
    try:
        mail_actions_path.relative_to(l2_root.resolve())
    except ValueError as exc:
        raise DeployError("Catalog outbound-mail actions escape the exact L2 source") from exc
    if allow_empty_integration:
        mail_actions_digest = sha256_bytes(canonical_bytes([]))
        mail_action_arguments: tuple[str, ...] = ()
    else:
        if not mail_actions_path.is_file():
            raise DeployError("Catalog outbound-mail action authority is unavailable")
        mail_actions_digest = sha256_bytes(mail_actions_path.read_bytes())
        mail_action_arguments = ("--outbound-mail-actions", str(mail_actions_path))
    catalog_versions = catalog_release_versions(context, l2_root=l2_root)
    abandonment = state.get("catalogAbandonment")
    abandonment_evidence = state.get("catalogAbandonmentEvidenceDigest", "")
    operation_attempt_digest = ""
    if abandonment is not None:
        completed_evidence = completed_catalog_abandonment_evidence(abandonment)
        if abandonment_evidence and abandonment_evidence != completed_evidence:
            raise DeployError("Catalog preparation found conflicting abandonment evidence")
        abandonment_evidence = completed_evidence
    if abandonment_evidence:
        if not isinstance(abandonment_evidence, str):
            raise DeployError("Catalog preparation found malformed abandonment evidence")
        operation_attempt_digest = catalog_successor_attempt_digest(
            abandonment_evidence,
            context.bom["desired_bom_sha256"],
        )
    idempotency_key = catalog_input_idempotency_key(
        context.bom,
        start,
        l1_version=catalog_versions["l1"],
        l2_version=catalog_versions["l2"],
        integration_bindings_digest=bindings_digest,
        outbound_mail_actions_digest=mail_actions_digest,
        approval_publication_digest=approval_bundle_digest,
        operation_attempt_digest=operation_attempt_digest,
    )
    package_path = context.bom["app_installation"]["artifact"]["path"]
    attestation_path = context.bom["app_installation"]["publisher_attestation"]["path"]
    execute((
        "python3", str(REPOSITORY_ROOT / "scripts" / "render_serviceification_test_stack.py"),
        "catalog",
        "--bom", str(context.run_root / "resolved-bom.json"),
        "--app-packs-root", str(app_packs_root),
        "--runtime-host-compatibility", str(prepared["runtimeHostCompatibility"]),
        "--catalog-host-compatibility", str(prepared["catalogHostCompatibility"]),
        "--output", str(catalog_config),
        "--org-id", start["orgId"],
        "--workload-id", context.bom["catalog_consumers"]["handler_workload_id"],
        "--runtime-audience", start["runtimeAudience"],
        "--package-mount-path", f"/release/{package_path}",
        "--attestation-mount-path", f"/release/{attestation_path}",
        "--artifact-store-root", "/var/lib/nexuscrm/catalog",
        "--idempotency-key", idempotency_key,
        "--operation-kind", start["mode"],
        "--app-installation-id", start["appInstallationId"],
        "--frontend-receipt-output", str(frontend_receipt),
        "--expected-catalog-revision", str(start["expectedCatalogRevision"]),
        "--expected-security-epoch", str(start["expectedSecurityEpoch"]),
        "--publisher-valid-from", "2026-01-01T00:00:00Z",
        "--publisher-valid-until", "2036-01-01T00:00:00Z",
        "--schema-generation", str(start["schemaGeneration"]),
        "--engine-generation", str(start["engineGeneration"]),
        "--integration-bindings", str(bindings_path),
        *mail_action_arguments,
        "--phase-one-publication-command-id", new_uuid7(),
        "--approval-publication-bundle-path", approval_container_path,
        "--approval-publication-bundle-digest", approval_bundle_digest,
        *(("--refresh-active-release",) if start.get("refreshActiveRelease") else ()),
        *(("--allow-empty-integration",) if allow_empty_integration else ()),
    ))
    if not catalog_config.is_file() or catalog_config.is_symlink():
        raise DeployError("D3 Catalog renderer emitted no regular reconcile config")
    os.chmod(catalog_config, 0o400)
    os.chmod(bindings_path, 0o400)
    os.chmod(frontend_receipt, 0o400)

    artifacts = {
        "config": str(catalog_config),
        "configDigest": sha256_bytes(catalog_config.read_bytes()),
        "output": str(output_dir / "catalog-output.json"),
        "store": str(store_dir),
        "frontendReceipt": str(frontend_receipt),
        "frontendReceiptDigest": sha256_bytes(frontend_receipt.read_bytes()),
        "integrationBindings": str(bindings_path),
        "integrationBindingsDigest": bindings_digest,
        "outboundMailActionsDigest": mail_actions_digest,
        "approvalPublication": str(approval_bundle),
        "approvalPublicationDigest": approval_bundle_digest,
        "databaseEnv": str(db_env_file),
        "appPacksRoot": str(app_packs_root),
        "operationKind": start["mode"],
        "refreshActiveRelease": bool(start.get("refreshActiveRelease", False)),
        "catalogVersions": catalog_versions,
    }
    if not allow_empty_integration:
        artifacts["outboundMailActions"] = str(mail_actions_path)
    replace_state(
        context,
        expected_phase="runtime-applied",
        updates={"phase": "catalog-ready", "catalogArtifacts": artifacts},
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "catalog-ready",
        "runId": context.run_id,
        "catalogStart": state["catalogStart"],
        "catalogBefore": state.get("catalogBefore"),
        "catalogInputDigests": {
            key: value for key, value in artifacts.items() if key.endswith("Digest")
        },
    }


def validated_catalog_result(
    context: D3Context, output: Path, start: Mapping[str, Any],
) -> dict[str, Any]:
    if not output.is_file() or output.is_symlink():
        raise DeployError("Catalog reconcile completed without a regular output receipt")
    try:
        result = json.loads(output.read_text())
    except (OSError, json.JSONDecodeError) as exc:
        raise DeployError("Catalog reconcile output is not JSON") from exc
    try:
        validate_catalog_output_for_deployment(
            dict(context.bom), result,
            org_id=str(start["orgId"]),
            workload_id=context.bom["catalog_consumers"]["handler_workload_id"],
        )
    except ValidationFailure as exc:
        raise DeployError(f"Catalog reconcile output is not deployable: {exc}") from exc
    expected_revision = int(start["expectedCatalogRevision"]) + 1
    expected_security = int(start["expectedSecurityEpoch"]) + 1
    if (
        not isinstance(result, dict)
        or result.get("orgId") != start["orgId"]
        or result.get("catalogRevision") != expected_revision
        or result.get("securityEpoch") != expected_security
        or result.get("schemaGeneration") != start["schemaGeneration"]
        or result.get("engineGeneration") != start["engineGeneration"]
    ):
        raise DeployError("Catalog reconcile output does not advance the exact stopped authority")
    if (
        start["mode"] == "UPGRADE"
        and result.get("appInstallationId") != start["appInstallationId"]
    ):
        raise DeployError("Catalog UPGRADE changed the AppInstallation identity")
    return result


def catalog_config_publisher_trust(config: Mapping[str, Any]) -> CatalogPublisherTrust:
    release_import = config.get("releaseImport")
    publisher = release_import.get("publisher") if isinstance(release_import, dict) else None
    raw_keys = publisher.get("keys") if isinstance(publisher, dict) else None
    publisher_id = publisher.get("publisherId") if isinstance(publisher, dict) else None
    if publisher_id != "io.nexuscrm" or not isinstance(raw_keys, list) or not raw_keys:
        raise DeployError("sealed Catalog config has no complete publisher trust target")
    keys: list[CatalogPublisherKey] = []
    for index, raw in enumerate(raw_keys):
        if not isinstance(raw, dict) or set(raw) != {
            "keyId", "algorithm", "publicKeyBase64", "validFrom", "validUntil",
        }:
            raise DeployError(f"sealed Catalog publisher key[{index}] has an unknown shape")
        values = {key: raw[key] for key in raw}
        if not all(isinstance(value, str) and value for value in values.values()):
            raise DeployError(f"sealed Catalog publisher key[{index}] is incomplete")
        try:
            public_key = base64.b64decode(values["publicKeyBase64"], validate=True)
        except (ValueError, binascii.Error) as exc:
            raise DeployError(f"sealed Catalog publisher key[{index}] is not base64") from exc
        if base64.b64encode(public_key).decode("ascii") != values["publicKeyBase64"]:
            raise DeployError(f"sealed Catalog publisher key[{index}] is not canonical base64")
        valid_from = canonical_catalog_config_time(
            values["validFrom"], f"key[{index}] validFrom",
        )
        valid_until = canonical_catalog_config_time(
            values["validUntil"], f"key[{index}] validUntil",
        )
        if datetime.fromisoformat(valid_until[:-1] + "+00:00") <= datetime.fromisoformat(
            valid_from[:-1] + "+00:00",
        ):
            raise DeployError(
                f"sealed Catalog publisher key[{index}] validity window is invalid",
            )
        keys.append(CatalogPublisherKey(
            key_id=values["keyId"], algorithm=values["algorithm"],
            public_key_base64=values["publicKeyBase64"],
            valid_from=valid_from, valid_until=valid_until,
        ))
    keys.sort(key=lambda key: key.key_id)
    if len({key.key_id for key in keys}) != len(keys):
        raise DeployError("sealed Catalog publisher target contains duplicate key IDs")
    return CatalogPublisherTrust(
        publisher_id=publisher_id, ownership_epoch=0, active=True, keys=tuple(keys),
    )


CATALOG_PUBLISHER_TRANSITION_RECEIPT_FIELDS = {
    "status", "config", "configDigest", "output", "operationId", "publisherId",
    "expectedOwnershipEpoch", "expectedTrustDigest", "targetTrustDigest",
}


def parse_catalog_publisher_transition_fence(
    expected_epoch: str,
    expected_trust_digest: str,
) -> tuple[int, str] | None:
    if not expected_epoch and not expected_trust_digest:
        return None
    if (
        not expected_epoch.isascii()
        or not expected_epoch.isdecimal()
        or expected_epoch.startswith("0")
        or int(expected_epoch) <= 0
    ):
        raise DeployError("publisher transition expected epoch is not canonical")
    if not SHA256_RE.fullmatch(expected_trust_digest):
        raise DeployError("publisher transition expected trust digest is not canonical")
    return int(expected_epoch), expected_trust_digest


def require_catalog_publisher_transition_receipt(value: object) -> dict[str, Any]:
    if not isinstance(value, dict) or set(value) != CATALOG_PUBLISHER_TRANSITION_RECEIPT_FIELDS:
        raise DeployError("publisher transition receipt is malformed")
    return value


def apply_catalog_publisher_trust_transition(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> bool:
    """Converge one explicit, resumable publisher-key transition before import."""
    state = require_state(
        context, expected_phase={"catalog-ready", "catalog-in-progress"},
    )
    phase = str(state["phase"])
    artifacts = state.get("catalogArtifacts")
    if not isinstance(artifacts, dict):
        raise DeployError("publisher transition has no sealed Catalog artifacts")
    catalog_config_path = Path(str(artifacts.get("config", "")))
    if (
        catalog_config_path.is_symlink()
        or not catalog_config_path.is_file()
        or sha256_bytes(catalog_config_path.read_bytes()) != artifacts.get("configDigest")
    ):
        raise DeployError("publisher transition lost the sealed Catalog config")
    target = catalog_config_publisher_trust(load_json_object(
        catalog_config_path, "sealed Catalog publisher transition config",
    ))

    receipt = state.get("catalogPublisherTrustTransition")
    if receipt is None:
        current = read_catalog_publisher_trust(context.envv)
        if current.digest() == target.digest():
            return False
        target_digest = target.digest()
        transition_config = {
            "schema": "nexuscrm.catalog-publisher-trust-transition/v1",
            "operationId": f"{context.run_id}-publisher-trust-{target_digest[7:19]}",
            "publisherId": current.publisher_id,
            "expectedOwnershipEpoch": str(current.ownership_epoch),
            "expectedActiveKeys": [key.config_value() for key in current.keys],
            "targetActiveKeys": [key.config_value() for key in target.keys],
        }
        config_path = catalog_config_path.parent / "publisher-trust-transition-config.json"
        output_path = (
            Path(str(artifacts.get("output", ""))).parent
            / "publisher-trust-transition-result.json"
        )
        sealed_config = canonical_bytes(transition_config)
        if output_path.exists() or output_path.is_symlink():
            raise DeployError("publisher transition output exists before its intent receipt")
        if config_path.exists() or config_path.is_symlink():
            if (
                config_path.is_symlink()
                or not config_path.is_file()
                or config_path.read_bytes() != sealed_config
            ):
                raise DeployError(
                    "publisher transition pre-receipt config does not match sealed intent",
                )
            os.chmod(config_path, 0o400)
        else:
            pending_config = config_path.with_name(
                f".{config_path.name}.{new_uuid7()}.pending",
            )
            if pending_config.exists() or pending_config.is_symlink():
                raise DeployError("publisher transition staging path already exists")
            pending_config.write_bytes(sealed_config)
            os.chmod(pending_config, 0o400)
            os.replace(pending_config, config_path)
        receipt = {
            "status": "pending",
            "config": str(config_path),
            "configDigest": sha256_bytes(config_path.read_bytes()),
            "output": str(output_path),
            "operationId": transition_config["operationId"],
            "publisherId": current.publisher_id,
            "expectedOwnershipEpoch": str(current.ownership_epoch),
            "expectedTrustDigest": current.digest(),
            "targetTrustDigest": target_digest,
        }
        replace_state(
            context,
            expected_phase=phase,
            updates={"catalogPublisherTrustTransition": receipt},
        )

    receipt = require_catalog_publisher_transition_receipt(receipt)
    if receipt["targetTrustDigest"] != target.digest():
        if phase != "catalog-ready" or receipt["status"] != "pending":
            raise DeployError("publisher transition target disagrees with sealed Catalog config")
        fence = parse_catalog_publisher_transition_fence(
            str(receipt["expectedOwnershipEpoch"]),
            str(receipt["expectedTrustDigest"]),
        )
        if fence is None:
            raise DeployError("superseded publisher transition lost its authority fence")
        current = read_catalog_publisher_trust(context.envv)
        if (
            current.publisher_id != receipt["publisherId"]
            or current.ownership_epoch != fence[0]
            or current.digest() != fence[1]
        ):
            raise DeployError(
                "superseded publisher transition may have changed Catalog authority",
            )
        if catalog_retry_operation_id(
            context.envv,
            load_json_object(
                catalog_config_path, "sealed Catalog publisher transition config",
            ),
        ):
            raise DeployError(
                "superseded publisher transition refuses an existing Catalog operation",
            )
        replace_state(
            context,
            expected_phase=phase,
            updates={"catalogPublisherTrustTransition": None},
        )
        return apply_catalog_publisher_trust_transition(context, execute=execute)
    if receipt["status"] == "completed":
        current = read_catalog_publisher_trust(context.envv)
        if (
            current.ownership_epoch != int(receipt["expectedOwnershipEpoch"]) + 1
            or current.digest() != receipt["targetTrustDigest"]
        ):
            raise DeployError("completed publisher transition disagrees with Catalog authority")
        return True
    if receipt["status"] != "pending":
        raise DeployError("publisher transition receipt has an unknown status")

    config_path = Path(str(receipt["config"]))
    output_path = Path(str(receipt["output"]))
    if (
        config_path.is_symlink()
        or not config_path.is_file()
        or sha256_bytes(config_path.read_bytes()) != receipt["configDigest"]
        or output_path.is_symlink()
    ):
        raise DeployError("sealed publisher transition input changed")
    db_env_file = Path(str(artifacts.get("databaseEnv", "")))
    if not db_env_file.is_file() or db_env_file.is_symlink() or db_env_file.stat().st_mode & 0o077:
        raise DeployError("publisher transition database credential file is unavailable")

    current = read_catalog_publisher_trust(context.envv)
    current_digest = current.digest()
    expected_epoch = int(receipt["expectedOwnershipEpoch"])
    if current.ownership_epoch == expected_epoch and current_digest == receipt["expectedTrustDigest"]:
        if output_path.exists():
            raise DeployError("publisher transition produced output before its authority changed")
    elif current.ownership_epoch == expected_epoch + 1 and current_digest == receipt["targetTrustDigest"]:
        durable = read_catalog_publisher_transition_receipt(
            context.envv, str(receipt["operationId"]),
        )
        if durable != {
            "operationId": receipt["operationId"],
            "publisherId": receipt["publisherId"],
            "expectedOwnershipEpoch": receipt["expectedOwnershipEpoch"],
            "ownershipEpoch": str(expected_epoch + 1),
            "previousTrustDigest": receipt["expectedTrustDigest"],
            "activeTrustDigest": receipt["targetTrustDigest"],
        }:
            raise DeployError(
                "publisher transition target authority has no exact durable operation receipt",
            )
    else:
        raise DeployError("pending publisher transition disagrees with Catalog authority")

    if not output_path.exists():
        container = container_name(context.run_id, "catalog-publisher-trust")
        execute((
            "docker", "run", "--rm", "--name", container,
            "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
            "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
            "--env-file", str(db_env_file),
            "--mount", f"type=bind,src={config_path},dst=/etc/nexuscrm/publisher-trust.json,readonly",
            "--mount", f"type=bind,src={output_path.parent},dst=/output",
            context.images["catalog_reconcile"],
            "publisher-trust", "-config", "/etc/nexuscrm/publisher-trust.json",
            "-output", "/output/publisher-trust-transition-result.json",
        ))
    output = load_json_object(output_path, "Catalog publisher transition output")
    if (
        output.get("schema") != "nexuscrm.catalog-publisher-trust-transition-result/v1"
        or output.get("operationId") != receipt["operationId"]
        or output.get("publisherId") != receipt["publisherId"]
        or output.get("ownershipEpoch") != str(expected_epoch + 1)
        or output.get("previousTrustDigest") != receipt["expectedTrustDigest"]
        or output.get("activeTrustDigest") != receipt["targetTrustDigest"]
    ):
        raise DeployError("Catalog publisher transition output does not match sealed intent")
    current = read_catalog_publisher_trust(context.envv)
    if current.ownership_epoch != int(output["ownershipEpoch"]) or current.digest() != output["activeTrustDigest"]:
        raise DeployError("Catalog publisher transition output disagrees with database authority")
    os.chmod(output_path, 0o400)
    replace_state(
        context,
        expected_phase=phase,
        updates={"catalogPublisherTrustTransition": {**receipt, "status": "completed"}},
    )
    return True


def apply_catalog(
    context: D3Context,
    *,
    execute: Callable[[Sequence[str]], None] = default_execute,
) -> dict[str, Any]:
    state = require_state(
        context, expected_phase={"catalog-ready", "catalog-in-progress"},
    )
    start = require_catalog_start(state)
    before = require_catalog_before(state) if start["mode"] == "UPGRADE" else None
    artifacts = state.get("catalogArtifacts")
    if not isinstance(artifacts, dict):
        raise DeployError("catalog-ready state has no sealed Catalog inputs")
    if artifacts.get("operationKind") != start["mode"]:
        raise DeployError("sealed Catalog operation kind changed")
    if bool(artifacts.get("refreshActiveRelease", False)) != bool(
        start.get("refreshActiveRelease", False)
    ):
        raise DeployError("sealed Catalog active-release refresh mode changed")
    required_paths = {
        "config": "configDigest",
        "frontendReceipt": "frontendReceiptDigest",
        "integrationBindings": "integrationBindingsDigest",
        "approvalPublication": "approvalPublicationDigest",
    }
    for path_key, digest_key in required_paths.items():
        path = Path(str(artifacts.get(path_key, "")))
        if not path.is_file() or path.is_symlink():
            raise DeployError(f"sealed Catalog input disappeared: {path_key}")
        if sha256_bytes(path.read_bytes()) != artifacts.get(digest_key):
            raise DeployError(f"sealed Catalog input changed: {path_key}")
    catalog_config = Path(str(artifacts["config"]))
    catalog_output = Path(str(artifacts["output"]))
    store_dir = Path(str(artifacts["store"]))
    db_env_file = Path(str(artifacts["databaseEnv"]))
    app_packs_root = Path(str(artifacts["appPacksRoot"]))
    approval_publication = Path(str(artifacts["approvalPublication"]))
    for path, label in (
        (store_dir, "Catalog snapshot store"),
        (app_packs_root, "exact AppRelease checkout"),
    ):
        if not path.is_dir() or path.is_symlink():
            raise DeployError(f"{label} disappeared before Catalog reconcile")
    if not db_env_file.is_file() or db_env_file.is_symlink() or db_env_file.stat().st_mode & 0o077:
        raise DeployError("Catalog database credential file is missing or disclosed")

    if state["phase"] == "catalog-ready":
        if read_optional_catalog_authority(context.envv) != before:
            raise DeployError("Catalog authority changed after D3 quiescence")

    if start["mode"] == "UPGRADE":
        apply_catalog_publisher_trust_transition(context, execute=execute)

    if state["phase"] == "catalog-ready":
        # catalog-in-progress means the resumable publisher transition has
        # completed (or was already unnecessary), never merely that its
        # precondition was observed before an interrupted process checkpoint.
        replace_state(
            context,
            expected_phase="catalog-ready",
            updates={"phase": "catalog-in-progress"},
        )

    catalog_container = container_name(context.run_id, "catalog-reconcile")
    execute((
        "docker", "run", "--rm", "--name", catalog_container,
        "--label", f"{CONTAINER_LABEL_KEY}={CONTAINER_LABEL_VALUE}",
        "--network", "host", "--user", f"{os.getuid()}:{os.getgid()}",
        "--env-file", str(db_env_file),
        "--mount", f"type=bind,src={catalog_config},dst=/etc/nexuscrm/catalog-config.json,readonly",
        "--mount", f"type=bind,src={approval_publication},dst=/etc/nexuscrm/approval-publication.json,readonly",
        "--mount", f"type=bind,src={app_packs_root},dst=/release,readonly",
        "--mount", f"type=bind,src={store_dir},dst=/var/lib/nexuscrm/catalog",
        "--mount", f"type=bind,src={catalog_output.parent},dst=/output",
        context.images["catalog_reconcile"],
        "-config", "/etc/nexuscrm/catalog-config.json",
        "-output", "/output/catalog-output.json",
    ))
    catalog_result = validated_catalog_result(context, catalog_output, start)
    current = read_catalog_authority(context.envv)
    if (
        current.app_installation_id != catalog_result["appInstallationId"]
        or current.org_id != catalog_result["orgId"]
        or current.active_release_digest != catalog_result["appReleaseDigest"]
        or current.catalog_revision != catalog_result["catalogRevision"]
        or current.runtime_audience != start["runtimeAudience"]
        or current.security_epoch != catalog_result["securityEpoch"]
        or current.schema_generation != catalog_result["schemaGeneration"]
        or current.engine_generation != catalog_result["engineGeneration"]
    ):
        raise DeployError("Catalog database authority does not match the exact reconcile receipt")
    os.chmod(catalog_output, 0o400)
    next_state = replace_state(
        context,
        expected_phase="catalog-in-progress",
        updates={
            "phase": "catalog-applied",
            "catalogAfter": {
                "appInstallationId": catalog_result["appInstallationId"],
                "orgId": catalog_result["orgId"],
                "catalogRevision": catalog_result["catalogRevision"],
                "runtimeAudience": current.runtime_audience,
                "securityEpoch": catalog_result["securityEpoch"],
                "activePublicationDigest": current.active_publication_digest,
                "catalogSnapshotDigest": catalog_result.get("catalogSnapshotDigest"),
                "schemaGeneration": catalog_result.get("schemaGeneration"),
                "engineGeneration": catalog_result.get("engineGeneration"),
            },
        },
    )
    return {
        "schema": EVIDENCE_SCHEMA,
        "stage": "catalog-applied",
        "runId": context.run_id,
        "catalogBefore": next_state["catalogBefore"],
        "catalogAfter": next_state["catalogAfter"],
    }


def refresh_semantically_identical_approval_publication_for_catalog_resume(
    context: D3Context,
) -> bool:
    """Rebuild failed Catalog inputs when only canonical bundle bytes changed."""
    state = require_state(context, expected_phase="catalog-in-progress")
    if catalog_implementation_retry_active(context, state):
        # supersede_unactivated_catalog_candidate already proved that only the
        # one-shot Catalog implementation changed and retained the sealed
        # inputs/operation. Approval-byte repair is a different recovery mode:
        # it requires pre-operation authority and must not run after a valid
        # BeginOperation merely because both modes resume catalog-in-progress.
        return False
    start = require_catalog_start(state)
    before = require_catalog_before(state) if start["mode"] == "UPGRADE" else None
    if read_optional_catalog_authority(context.envv) != before:
        raise DeployError(
            "Approval publication recovery refuses changed Catalog authority"
        )
    prepared = state.get("prepared")
    artifacts = state.get("catalogArtifacts")
    if not isinstance(prepared, dict) or not isinstance(artifacts, dict):
        raise DeployError("Approval publication recovery has no sealed Catalog inputs")

    sealed = Path(str(prepared.get("approvalPublication", "")))
    sealed_digest = str(prepared.get("approvalPublicationDigest", ""))
    if (
        sealed.is_symlink()
        or not sealed.is_file()
        or sha256_bytes(sealed.read_bytes()) != sealed_digest
        or artifacts.get("approvalPublication") != str(sealed)
        or artifacts.get("approvalPublicationDigest") != sealed_digest
    ):
        raise DeployError("Approval publication recovery cannot reopen the sealed bundle")
    source, source_digest = validate_candidate_approval_publication(context)
    if source_digest == sealed_digest:
        return False
    try:
        sealed_value = json.loads(sealed.read_bytes())
        source_value = json.loads(source.read_bytes())
    except (OSError, json.JSONDecodeError) as exc:
        raise DeployError("Approval publication recovery input is invalid") from exc
    if sealed_value != source_value:
        raise DeployError(
            "Approval publication recovery refuses a semantic authority change"
        )
    output = Path(str(artifacts.get("output", "")))
    if output.exists() and (output.is_symlink() or not output.is_file() or output.stat().st_size):
        raise DeployError(
            "Approval publication recovery refuses a completed or uncertain Catalog output"
        )

    pending = sealed.with_name(f".{sealed.name}.{new_uuid7()}.pending")
    if pending.exists() or pending.is_symlink():
        raise DeployError("Approval publication recovery staging path already exists")
    pending.write_bytes(source.read_bytes())
    os.chmod(pending, 0o400)
    os.replace(pending, sealed)
    replacement = dict(prepared)
    replacement["approvalPublicationDigest"] = source_digest
    replace_state(
        context,
        expected_phase="catalog-in-progress",
        updates={
            "phase": "runtime-applied",
            "prepared": replacement,
            "approvalPublicationRecovery": {
                "reason": "semantic-content-identical-canonical-byte-repair",
                "supersededDigest": sealed_digest,
                "replacementDigest": source_digest,
            },
        },
    )
    return True


def catalog_implementation_retry_active(
    context: D3Context,
    state: Mapping[str, Any],
) -> bool:
    receipt = state.get("catalogImplementationRetry")
    if receipt is None:
        return False
    expected_keys = {"operationId", "previousImage", "replacementImage"}
    if not isinstance(receipt, dict) or set(receipt) != expected_keys:
        raise DeployError("Catalog implementation retry receipt is malformed")
    operation_id = receipt.get("operationId")
    previous = receipt.get("previousImage")
    replacement = receipt.get("replacementImage")
    expected_replacement = context.bom["jobs"]["catalog_reconcile"]["digest"]
    if (
        not isinstance(operation_id, str)
        or not isinstance(previous, str)
        or not SHA256_RE.fullmatch(previous)
        or not isinstance(replacement, str)
        or not SHA256_RE.fullmatch(replacement)
        or previous == replacement
        or replacement != expected_replacement
    ):
        raise DeployError("Catalog implementation retry receipt disagrees with the exact BOM")
    return True


def recover_catalog_refresh_mode(context: D3Context) -> bool:
    """Reseal a pre-operation Catalog request whose refresh mode is stale."""
    state = require_state(context, expected_phase="catalog-in-progress")
    start = require_catalog_start(state)
    if start["mode"] != "UPGRADE":
        return False
    before = require_catalog_before(state)
    target_release = context.bom["app_installation"]["app_release_digest"]
    refresh_active_release = before.active_release_digest == target_release
    if bool(start.get("refreshActiveRelease")) == refresh_active_release:
        return False
    if read_optional_catalog_authority(context.envv) != before:
        raise DeployError("Catalog refresh-mode recovery refuses changed authority")
    artifacts = state.get("catalogArtifacts")
    if not isinstance(artifacts, dict):
        raise DeployError("Catalog refresh-mode recovery has no sealed inputs")
    config_path = Path(str(artifacts.get("config", "")))
    config_digest = str(artifacts.get("configDigest", ""))
    if (
        config_path.is_symlink()
        or not config_path.is_file()
        or sha256_bytes(config_path.read_bytes()) != config_digest
    ):
        raise DeployError("Catalog refresh-mode recovery lost its sealed config")
    config = load_json_object(config_path, "sealed Catalog refresh-mode config")
    if catalog_retry_operation_id(context.envv, config):
        raise DeployError(
            "Catalog refresh-mode recovery refuses an existing Catalog operation"
        )
    output = Path(str(artifacts.get("output", "")))
    if output.exists() and (output.is_symlink() or not output.is_file() or output.stat().st_size):
        raise DeployError(
            "Catalog refresh-mode recovery refuses a completed or uncertain output"
        )
    replace_state(
        context,
        expected_phase="catalog-in-progress",
        updates={
            "phase": "runtime-applied",
            "catalogStart": {
                "mode": "UPGRADE",
                "refreshActiveRelease": refresh_active_release,
            },
            "catalogRefreshModeRecovery": {
                "reason": "refresh-mode-recomputed-from-active-and-target-release",
                "activeReleaseDigest": before.active_release_digest,
                "targetReleaseDigest": target_release,
                "refreshActiveRelease": refresh_active_release,
            },
        },
    )
    return True


def rebind_exact_source_paths(
    context: D3Context,
    state: Mapping[str, Any],
    *,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
) -> dict[str, Any]:
    if state.get("phase") in {
        "governed-reference-data-applied", "serving-inputs-ready",
        "worker-activation-in-progress", "worker-owned",
        "serving-start-in-progress", "serving",
    }:
        # Source checkouts are inputs through Catalog publication and the
        # governed L2 reference-data apply. Serving consumes their sealed
        # receipts and immutable image/BOM digests; binding it back to a
        # historical checkout couples unrelated L0 implementations to
        # already-published L1/L2 artifacts.
        return dict(state)
    exact_sources = state.get("exactSources")
    if not isinstance(exact_sources, dict):
        return dict(state)
    app_packs_root = require_exact_checkout(
        app_packs_root, str(exact_sources.get("appPacksCommit", "")), "AppRelease",
    )
    core_root = require_exact_checkout(
        core_root, str(exact_sources.get("coreCommit", "")), "Core",
    )
    l2_root = require_exact_checkout(
        l2_root, str(exact_sources.get("l2Commit", "")), "L2 metadata",
    )
    rebound = {
        **exact_sources,
        "appPacksRoot": str(app_packs_root),
        "coreRoot": str(core_root),
        "l2Root": str(l2_root),
    }
    if rebound == exact_sources:
        return dict(state)
    return replace_state(
        context, expected_phase=str(state["phase"]), updates={"exactSources": rebound},
    )


def converge_deployment(
    context: D3Context,
    *,
    app_packs_root: Path,
    core_root: Path,
    l2_root: Path,
    preserve_current_database: bool = False,
) -> dict[str, Any]:
    if context.state_file.exists() or context.state_file.is_symlink():
        active = read_state(context.state_file)
        active_run_id = str(active.get("runId", ""))
        if active_run_id != context.run_id:
            active_phase = str(active.get("phase", ""))
            if active_phase in {
                "quiesced", "runtime-apply-in-progress", "runtime-candidate-applied",
                "runtime-applied", "catalog-ready", "catalog-in-progress",
            }:
                supersede_unactivated_catalog_candidate(
                    context, active=active, expected_active_run_id=active_run_id,
                )
            elif active_phase in {
                "catalog-applied", "governed-reference-data-in-progress",
                "governed-reference-data-applied", "serving-inputs-ready",
                "worker-activation-in-progress", "worker-owned",
                "serving-start-in-progress", "serving",
                "service-image-replace-in-progress",
            }:
                # A serving deployment is the rollback owner, not a terminal
                # lock on the slot. Route a signed BOM through the existing
                # successor validator. The same BOM is accepted only to retire
                # transient service-image state; otherwise this remains one
                # compatible serving refresh or stopped Catalog upgrade.
                supersede_serving_implementation_candidate(
                    context,
                    active=active,
                    expected_active_run_id=active_run_id,
                    core_root=core_root,
                    app_packs_root=app_packs_root,
                    l2_root=l2_root,
                )
            else:
                raise DeployError(
                    "this slot already has deployment "
                    f"{active_run_id!r} in phase {active_phase!r}; "
                    "resume that deployment_run_id; database refresh is a separate "
                    "operator transaction"
                )
    if not context.state_file.exists() and not context.state_file.is_symlink():
        capture(context)
    state = require_state(context)
    if state.get("runId") != context.run_id:
        raise DeployError(
            f"active D3 transition belongs to run {state.get('runId')}, not {context.run_id}"
        )
    state = rebind_exact_source_paths(
        context, state, app_packs_root=app_packs_root,
        core_root=core_root, l2_root=l2_root,
    )
    for _ in range(16):
        current = read_state(context.state_file)
        phase = str(current.get("phase", ""))
        if (
            isinstance(current.get("baselineReset"), dict)
            and current.get("testIdentityAligned") is None
        ):
            ensure_test2_inspection_identity(context, core_root)
            continue
        if phase == "captured":
            prepare_artifacts(
                context, app_packs_root=app_packs_root,
                core_root=core_root, l2_root=l2_root,
            )
        elif phase in {"catalog-successor-captured", "runtime-successor-captured"}:
            prepare_artifacts(
                context, app_packs_root=app_packs_root,
                core_root=core_root, l2_root=l2_root,
            )
        elif phase == "artifacts-ready" and isinstance(
            current.get("productionCloneResetRequest"), dict,
        ):
            reset_test2_production_baseline(context)
        elif phase == "artifacts-ready" and preserve_current_database:
            preserve_current_test2_database(context)
        elif (
            phase in BASELINE_RESET_ELIGIBLE_PHASES
            and not isinstance(current.get("baselineReset"), dict)
        ):
            reset_test2_production_baseline(context)
        elif phase in {"baseline-reset", "quiesce-in-progress"}:
            quiesce(context)
        elif phase == "runtime-apply-in-progress":
            refresh_incompatible_deployment_tools(context)
            apply_stopped_upgrade_runtime(context)
        elif phase == "runtime-candidate-applied":
            apply_stopped_upgrade_runtime(context)
        elif phase == "quiesced":
            apply_stopped_upgrade_runtime(context)
        elif phase == "runtime-applied":
            if apply_pending_catalog_abandonment(context):
                continue
            prepare_catalog(context)
        elif phase == "catalog-in-progress":
            if refresh_semantically_identical_approval_publication_for_catalog_resume(context):
                continue
            if recover_catalog_refresh_mode(context):
                continue
            apply_catalog(context)
        elif phase == "catalog-ready":
            apply_catalog(context)
        elif phase == "catalog-applied":
            apply_governed_reference_data(context)
        elif phase == "governed-reference-data-in-progress":
            apply_governed_reference_data(context)
        elif phase == "governed-reference-data-applied":
            prepare_serving_inputs(context)
        elif phase in {"serving-inputs-ready", "worker-activation-in-progress"}:
            activate_worker_ownership(context)
        elif phase in {"worker-owned", "serving-start-in-progress"}:
            start_serving(context)
        elif phase == "serving":
            if recover_incomplete_serving_inventory(context, current):
                continue
            return verify_serving(context)
        else:
            raise DeployError(f"D3 deployment cannot converge unknown phase: {phase}")
    raise DeployError("D3 deployment exceeded the bounded convergence stage count")


def print_json(value: Mapping[str, Any]) -> None:
    print(json.dumps(value, sort_keys=True, separators=(",", ":")))


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command", choices=(
            "preflight", "capture", "prepare-artifacts", "quiesce",
            "apply-stopped-upgrade-runtime",
            "prepare-catalog", "apply-catalog", "prepare-serving-inputs",
            "activate-worker-ownership",
            "start-serving",
            "deploy",
            "verify-serving", "exercise-restart", "reconfigure-worker-role",
            "replace-worker-image",
            "replace-serving-images",
            "rotate-serving-tls",
            "replace-frontend-image",
            "status", "status-active",
        ),
        help="preflight is read-only; capture durably records the exact rollback boundary",
    )
    parser.add_argument("--env-file", required=True)
    parser.add_argument("--resolved-bom")
    parser.add_argument("--run-id", required=True)
    parser.add_argument("--app-packs-root")
    parser.add_argument("--core-root")
    parser.add_argument("--l2-root")
    parser.add_argument("--role")
    parser.add_argument("--parallelism", type=int)
    parser.add_argument("--poll-interval")
    parser.add_argument("--runtime-api-image")
    parser.add_argument("--runtime-worker-image")
    parser.add_argument("--runtime-worker-schema-carrier-image")
    parser.add_argument("--integration-service-image")
    parser.add_argument("--sfcrm-app-service-image")
    parser.add_argument("--frontend-image")
    parser.add_argument("--runtime-api-profiling-enabled", choices=("true", "false"))
    parser.add_argument("--runtime-api-sql-diagnostics-enabled", choices=("true", "false"))
    parser.add_argument("--runtime-api-db-pool-size", type=int)
    parser.add_argument("--service-image-failure-evidence")
    parser.add_argument(
        "--preserve-current-database",
        action="store_true",
        help="use the current accepted Test2 database without another branch reset",
    )
    args = parser.parse_args()
    try:
        if args.command == "status-active":
            context = load_active_transition_context(
                Path(args.env_file),
                args.run_id,
                allow_legacy_reset_receipt=True,
            )
        elif args.command in {
            "reconfigure-worker-role",
            "replace-worker-image",
            "replace-serving-images",
            "rotate-serving-tls",
            "replace-frontend-image",
        }:
            # These are mutations of one already-active deployment. Its sealed
            # state owns the exact historical BOM; requiring the workflow to
            # reconstruct or pass a second path would create another authority.
            context = load_active_transition_context(
                Path(args.env_file),
                args.run_id,
            )
        else:
            if not args.resolved_bom:
                raise DeployError(f"{args.command} requires --resolved-bom")
            context = load_context(Path(args.env_file), Path(args.resolved_bom), args.run_id)
        if args.command == "preflight":
            print_json(preflight(context))
        elif args.command == "capture":
            print_json(capture(context))
        elif args.command == "prepare-artifacts":
            if not args.app_packs_root or not args.core_root or not args.l2_root:
                raise DeployError(
                    "prepare-artifacts requires --app-packs-root, --core-root, and --l2-root",
                )
            print_json(prepare_artifacts(
                context,
                app_packs_root=Path(args.app_packs_root),
                core_root=Path(args.core_root),
                l2_root=Path(args.l2_root),
            ))
        elif args.command == "quiesce":
            print_json(quiesce(context))
        elif args.command == "apply-stopped-upgrade-runtime":
            print_json(apply_stopped_upgrade_runtime(context))
        elif args.command == "prepare-catalog":
            print_json(prepare_catalog(context))
        elif args.command == "apply-catalog":
            print_json(apply_catalog(context))
        elif args.command == "prepare-serving-inputs":
            print_json(prepare_serving_inputs(context))
        elif args.command == "activate-worker-ownership":
            print_json(activate_worker_ownership(context))
        elif args.command == "start-serving":
            print_json(start_serving(context))
        elif args.command == "deploy":
            if not args.app_packs_root or not args.core_root or not args.l2_root:
                raise DeployError(
                    "deploy requires --app-packs-root, --core-root, and --l2-root",
                )
            print_json(converge_deployment(
                context,
                app_packs_root=Path(args.app_packs_root),
                core_root=Path(args.core_root),
                l2_root=Path(args.l2_root),
                preserve_current_database=args.preserve_current_database,
            ))
        elif args.command == "verify-serving":
            print_json(verify_serving(context))
        elif args.command == "exercise-restart":
            if not args.role:
                raise DeployError("exercise-restart requires --role")
            print_json(exercise_restart(context, args.role))
        elif args.command == "reconfigure-worker-role":
            if not args.role or (args.parallelism is None) == (args.poll_interval is None):
                raise DeployError(
                    "reconfigure-worker-role requires --role and exactly one runtime setting",
                )
            print_json(reconfigure_worker_role(
                context, args.role,
                parallelism=args.parallelism,
                poll_interval=args.poll_interval,
            ))
        elif args.command == "replace-worker-image":
            if not args.runtime_worker_image or not args.runtime_worker_schema_carrier_image:
                raise DeployError(
                    "replace-worker-image requires --runtime-worker-image and "
                    "--runtime-worker-schema-carrier-image",
                )
            print_json(replace_worker_image(
                context,
                args.runtime_worker_image,
                args.runtime_worker_schema_carrier_image,
            ))
        elif args.command == "replace-serving-images":
            print_json(replace_serving_images(
                context,
                runtime_api_image=args.runtime_api_image,
                integration_service_image=args.integration_service_image,
                sfcrm_app_service_image=args.sfcrm_app_service_image,
                runtime_api_profiling_enabled=(
                    args.runtime_api_profiling_enabled == "true"
                    if args.runtime_api_profiling_enabled is not None else None
                ),
                runtime_api_sql_diagnostics_enabled=(
                    args.runtime_api_sql_diagnostics_enabled == "true"
                    if args.runtime_api_sql_diagnostics_enabled is not None else None
                ),
                runtime_api_db_pool_size=args.runtime_api_db_pool_size,
                failure_evidence=(
                    Path(args.service_image_failure_evidence)
                    if args.service_image_failure_evidence else None
                ),
            ))
        elif args.command == "rotate-serving-tls":
            print_json(rotate_serving_tls(context))
        elif args.command == "replace-frontend-image":
            if not args.frontend_image:
                raise DeployError("replace-frontend-image requires --frontend-image")
            print_json(replace_frontend_image(
                context,
                frontend_image=args.frontend_image,
            ))
        else:
            print_json(read_state(context.state_file))
    except (DeployError, OSError, subprocess.CalledProcessError) as exc:
        print(f"D3 test2 {args.command} failed: {exc}", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(
        "Shared D3 engine: invoke the Test1/Test3, Staging or Prod environment adapter. "
        "Use standalone workflows for Test2."
    )