#!/usr/bin/env python3 from __future__ import annotations import argparse import hashlib import json import os import re import subprocess import sys import time from pathlib import Path import common import hubspot_prod_contract REPO_ROOT = Path(__file__).resolve().parents[3] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) from scripts.connected_mail_oauth import ( # noqa: E402 CONNECTED_MAIL_OAUTH_ENV_KEYS, connected_mail_oauth_env, ) from scripts.salesforce_migration.file_history_contract import ( # noqa: E402 SALESFORCE_HISTORY_ARCHIVE_BUCKET, SALESFORCE_HISTORY_ARCHIVE_PREFIX, SALESFORCE_HISTORY_ARCHIVE_REGION, ) EXPECTED_PROD_TIDB_PORT = "4000" EXPECTED_PROD_TIDB_DATABASE = "nexuscrm" RELEASE_CANDIDATE_CONTRACT_VERSION = "release-candidate/v3" EXPECTED_PROD_SMTP_HOST = "email-smtp.us-east-1.amazonaws.com" EXPECTED_PROD_SMTP_PORT = "587" EXPECTED_PROD_SMTP_FROM = "noreply@mail.nexuscrm.pingcap.com" PROD_HUBSPOT_WORKFLOW_ACTION_POLICY = "must_complete" PROD_HUBSPOT_WORKFLOW_ACTION_REPLAY_RPS = "4" PROD_RESET_NO_ACTIVE_RELEASE = "__none__" PROD_NGINX_CONF_PATH = Path("/etc/nginx/conf.d/nexuscrm-prod.conf") BACKEND_PROXY_RE = re.compile(r"proxy_pass\s+http://127\.0\.0\.1:(\d+)") DEFAULT_PROD_BACKEND_PORT_SLOTS = ("3103", "3104", "3105", "3106") PROD_FILE_STORAGE_BUCKET = "nexus-crm-files-prod" PROD_FILE_STORAGE_REGION = "us-west-2" PROD_FILE_STORAGE_PREFIX = "prod/customer-files/v1" DEFAULT_PROD_SALESFORCE_FILE_STORAGE_BUCKET = SALESFORCE_HISTORY_ARCHIVE_BUCKET DEFAULT_PROD_SALESFORCE_FILE_STORAGE_REGION = SALESFORCE_HISTORY_ARCHIVE_REGION PROD_SALESFORCE_FILE_STORAGE_PREFIX = SALESFORCE_HISTORY_ARCHIVE_PREFIX RETIRED_PROD_CONTROL_PLANE_RUNTIME_KEYS = frozenset( { "TIDB_CLOUD_API_BASE_URL", "TIDB_CLOUD_PUBLIC_KEY", "TIDB_CLOUD_PRIVATE_KEY", "TIDB_CLOUD_CLUSTER_ID", "TIDB_CLOUD_BRANCH_ID", "PROD_RECOVERY_TIDB_CLOUD_BRANCH_ID", "PROD_RECOVERY_TIDB_PASSWORD", } ) def require_prod_salesforce_file_storage_iam_authority( *, bucket: str, region: str, ) -> None: if ( bucket != DEFAULT_PROD_SALESFORCE_FILE_STORAGE_BUCKET or region != DEFAULT_PROD_SALESFORCE_FILE_STORAGE_REGION ): raise common.DeployError( "Prod Salesforce file-storage authority is not bound by " "prod_file_storage_iam_policy.json: " f"bucket={bucket!r} region={region!r}" ) RETAINED_RELEASE_CONFIG_BINDINGS = { "RELEASE_TAG": "release_tag", "DEPLOYMENT_AUTHORITY_MODE": "authority", "RELEASE_REFS_SOURCE": "release_refs_source", "TAG_TARGET_COMMIT": "tag_target_commit", "RELEASE_EXECUTOR_COMMIT": "executor_commit", "LINEAGE_PATH": "lineage_path", "DOWNSTREAM_REF": "downstream_ref", "UPSTREAM_REF": "upstream_ref", "OFFICIAL_APP_PACKS_REF": "official_app_packs_ref", "STAGING_RELEASE_RUN_ID": "source_run_id", "PROD_CANDIDATE_RESET_BRANCH": "candidate_reset_branch", "PROD_EXPECTED_FINAL_PACK_HASH": "final_pack_hash", "PROD_CANDIDATE_BUNDLE_ARTIFACT_NAME": "candidate_bundle_artifact_name", "PROD_CANDIDATE_BUNDLE_ARCHIVE_RELATIVE_PATH": ( "candidate_bundle_archive_relative_path" ), "PROD_CANDIDATE_BUNDLE_ARCHIVE_SHA256": "candidate_bundle_archive_sha256", "PROD_CANDIDATE_BUNDLE_MANIFEST_SHA256": "candidate_bundle_manifest_sha256", "PROD_CANDIDATE_CONNECT_MCP_URL": "candidate_connect_mcp_url", "PROD_PROMOTION_AUTHORITY_SHA256": "promotion_authority_sha256", "PROD_EXPECTED_APP_SHELL_CONFIG_SHA256": "app_shell_config_hash", "PROD_ORDINARY_PROMOTION_CONTRACT_SHA256": ( "ordinary_prod_promotion_contract_sha256" ), "PROD_ORDINARY_SHARED_STATE_DIGEST": "ordinary_prod_shared_state_digest", "PROD_ORDINARY_PROMOTION_ENFORCEMENT_STATUS": ( "ordinary_prod_promotion_enforcement_status" ), } SMTP_RUNTIME_KEYS = { "SMTP_HOST", "SMTP_PORT", "SMTP_USER", "SMTP_PASSWORD", "SMTP_FROM", "SMTP_TLS", } DEPLOYMENT_CONFIG = REPO_ROOT / "deployments" / "deployment-config.json" from scripts.connector_runtime_safety import ( # noqa: E402 SAFE_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS, SAFE_CONNECTOR_INBOUND_WORKERS_PER_PACK, validate_release_connector_inbound_max_processing_rows, validate_release_connector_inbound_workers_per_pack, ) from scripts.deployment_bundle import BUNDLE_CONTRACT_VERSION # noqa: E402 from scripts.release_setup_contract import ( # noqa: E402 require_prod_release_setup_evidence, ) from scripts.prod_promotion_contract import ( # noqa: E402 build_active_baseline_comparison, build_prod_promotion_contract_from_git, require_prod_baseline_receipt, require_prod_promotion_contract, ) from scripts.prod_migrated_baseline_receipt import ( # noqa: E402 require_bound_migrated_baseline_receipt, ) from scripts.promotion_authority import ( # noqa: E402 build_promotion_authority, require_tag_promotion_authority, ) def release_connector_inbound_workers_per_pack(raw_value: str | None) -> str: try: return validate_release_connector_inbound_workers_per_pack(raw_value, environment="multi_prod") except ValueError as exc: raise common.DeployError(str(exc)) from exc def release_connector_inbound_max_processing_rows(raw_value: str | None) -> str: try: return validate_release_connector_inbound_max_processing_rows(raw_value, environment="multi_prod") except ValueError as exc: raise common.DeployError(str(exc)) from exc def prod_connector_live_queue_stats_enabled(raw_value: str | None) -> str: value = raw_value.strip().lower() if raw_value is not None else "false" if value != "false": raise common.DeployError( "NEXUSCRM_CONNECTOR_LIVE_QUEUE_STATS_ENABLED is fixed for multi_prod as 'false'" ) return "false" def scim_inprocess_enabled(raw_value: str | None) -> str: value = (raw_value or "false").strip().lower() or "false" if value not in {"true", "false"}: raise common.DeployError(f"invalid NEXUSCRM_SCIM_INPROCESS_ENABLED={raw_value}") return value def prod_connector_webhook_ack_drop_endpoints(raw_value: str | None) -> str: if ( raw_value is not None and raw_value.strip() != hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS ): raise common.DeployError( "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS is fixed for multi_prod as " f"{hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS!r}" ) return hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS def prod_connector_webhook_ack_drop_timeout_ms(raw_value: str | None) -> str: if ( raw_value is not None and raw_value.strip() != hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS ): raise common.DeployError( "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS is fixed for multi_prod as " f"{hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS!r}" ) return hubspot_prod_contract.PROD_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--env", choices=["multi_prod"], required=True) parser.add_argument("--release-tag", required=True) parser.add_argument("--public-base-url", required=True) parser.add_argument("--healthcheck-host", required=True) parser.add_argument("--validate-retained-runtime-env-file", type=Path) parser.add_argument("--expected-executor-commit") return parser.parse_args() def update_env_file(path: Path, updates: dict[str, str], *, remove_keys: set[str] | None = None) -> None: remove_keys = remove_keys or set() lines = path.read_text().splitlines() if path.is_file() else [] seen: set[str] = set() out: list[str] = [] for line in lines: if not line or line.startswith("#") or "=" not in line: out.append(line) continue key, _ = line.split("=", 1) if key in remove_keys: seen.add(key) continue if key in updates: out.append(f"{key}={updates[key]}") seen.add(key) else: out.append(line) for key, value in updates.items(): if key not in seen: out.append(f"{key}={value}") path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.parent / f".{path.name}.tmp" tmp_path.write_text("\n".join(out) + "\n") tmp_path.chmod(0o600) os.replace(tmp_path, path) def deployment_root() -> Path: raw = os.environ.get("DEPLOYMENT_ROOT") if raw: return Path(raw).expanduser().resolve() return (Path.home() / "nexuscrm-deployment").resolve() def online_envs_root() -> Path: raw = os.environ.get("ONLINE_ENVS_ROOT") if raw: return Path(raw).expanduser().resolve() return deployment_root() / "online" / "envs" def runtime_env_file(env_name: str) -> Path: return online_envs_root() / env_name / "env" / "runtime.env" def release_root(env_name: str, release_tag: str) -> Path: return online_envs_root() / env_name / "releases" / release_tag def release_runtime_env_file(env_name: str, release_tag: str) -> Path: return release_root(env_name, release_tag) / "env" / "runtime.env" def current_release_tag_for_reset_source(env_root: Path) -> str: marker = env_root / "current.release-tag" if not marker.exists(): return PROD_RESET_NO_ACTIVE_RELEASE if not marker.is_file(): raise common.DeployError(f"prod current release marker is not a file: {marker}") text = marker.read_text().strip() if not text: raise common.DeployError(f"prod current release marker is empty: {marker}") try: payload = json.loads(text) except json.JSONDecodeError: release_tag = next( ( line.strip() for line in text.splitlines() if line.strip() and "=" not in line ), "", ) else: release_tag = ( str(payload.get("release_tag", "")).strip() if isinstance(payload, dict) else "" ) if not release_tag: raise common.DeployError(f"prod current release marker has no release tag: {marker}") return release_tag def current_release_marker_sha256_for_reset_source(env_root: Path) -> str: marker = env_root / "current.release-tag" if not marker.exists(): return PROD_RESET_NO_ACTIVE_RELEASE if not marker.is_file(): raise common.DeployError(f"prod current release marker is not a file: {marker}") return hashlib.sha256(marker.read_bytes()).hexdigest() def capture_prod_reset_source_identity( env_root: Path, *, nginx_conf_path: Path = PROD_NGINX_CONF_PATH, backend_port_slots: tuple[str, ...] = DEFAULT_PROD_BACKEND_PORT_SLOTS, ) -> dict[str, str]: env_root = env_root.expanduser().resolve() release_tag = current_release_tag_for_reset_source(env_root) marker_sha256 = current_release_marker_sha256_for_reset_source(env_root) nginx_conf_path = nginx_conf_path.expanduser().resolve() if release_tag == PROD_RESET_NO_ACTIVE_RELEASE: live_signals = common.prod_reset_source_live_signals( env_root, nginx_conf_path, backend_port_slots, ) if live_signals: raise common.DeployError( "prod current release marker is absent but live runtime signals remain: " + ", ".join(live_signals) ) return { "release_tag": release_tag, "marker_sha256": marker_sha256, "runtime_env_sha256": PROD_RESET_NO_ACTIVE_RELEASE, "backend_port": PROD_RESET_NO_ACTIVE_RELEASE, "nginx_conf_path": str(nginx_conf_path), } release_root = (env_root / "releases" / release_tag).resolve() expected_runtime_env = (release_root / "env" / "runtime.env").resolve() active_runtime_link = env_root / "current" / "env" / "runtime.env" if not active_runtime_link.is_symlink(): raise common.DeployError( f"prod active runtime env is not a symlink: {active_runtime_link}" ) active_runtime_env = active_runtime_link.resolve() if active_runtime_env != expected_runtime_env or not active_runtime_env.is_file(): raise common.DeployError( "prod active runtime env does not match current.release-tag: " f"got {active_runtime_env}, want {expected_runtime_env}" ) runtime_values = common.load_env_file(active_runtime_env) if str(runtime_values.get("RELEASE_TAG", "")).strip() != release_tag: raise common.DeployError("prod active runtime RELEASE_TAG does not match current.release-tag") if Path(str(runtime_values.get("ENV_ROOT", "")).strip()).expanduser().resolve() != release_root: raise common.DeployError("prod active runtime ENV_ROOT does not match current.release-tag") backend_port = str(runtime_values.get("BACKEND_PORT", "")).strip() if not backend_port.isdigit(): raise common.DeployError("prod active runtime has no valid BACKEND_PORT") metadata_path = env_root / "current" / "metadata" / "release.json" try: metadata = json.loads(metadata_path.read_text()) except (OSError, json.JSONDecodeError) as exc: raise common.DeployError(f"cannot read prod active release metadata {metadata_path}: {exc}") from exc if not isinstance(metadata, dict): raise common.DeployError(f"prod active release metadata is not an object: {metadata_path}") if str(metadata.get("release_tag", "")).strip() != release_tag: raise common.DeployError("prod active release metadata tag does not match current.release-tag") if Path(str(metadata.get("version_root", "")).strip()).expanduser().resolve() != release_root: raise common.DeployError("prod active release metadata root does not match current.release-tag") if ( Path(str(metadata.get("runtime_env_file", "")).strip()).expanduser().resolve() != expected_runtime_env ): raise common.DeployError("prod active release metadata runtime env does not match current.release-tag") try: nginx_content = nginx_conf_path.read_text() except OSError as exc: raise common.DeployError(f"cannot read prod nginx config {nginx_conf_path}: {exc}") from exc nginx_match = BACKEND_PROXY_RE.search(nginx_content) nginx_port = nginx_match.group(1) if nginx_match else "" if nginx_port != backend_port: raise common.DeployError( "prod active nginx backend does not match current runtime: " f"nginx={nginx_port!r} runtime={backend_port!r}" ) service = f"nexuscrm-multi-prod-slot-{backend_port}-backend.service" active = subprocess.run( ["systemctl", "is-active", "--quiet", service], check=False, ) if active.returncode != 0: raise common.DeployError(f"prod active backend service is not active: {service}") service_env = subprocess.run( ["systemctl", "show", service, "--property=EnvironmentFiles", "--value"], check=False, capture_output=True, text=True, ) if service_env.returncode != 0 or str(active_runtime_env) not in service_env.stdout.split(): raise common.DeployError("prod active backend service is not bound to the current runtime env") health = subprocess.run( ["curl", "-fsS", "--max-time", "5", f"http://127.0.0.1:{backend_port}/health"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) if health.returncode != 0: raise common.DeployError("prod active backend health check failed") return { "release_tag": release_tag, "marker_sha256": marker_sha256, "runtime_env_sha256": hashlib.sha256(active_runtime_env.read_bytes()).hexdigest(), "backend_port": backend_port, "nginx_conf_path": str(nginx_conf_path), } def build_deploy_id(env_name: str) -> str: stamp = subprocess.run( ["date", "-u", "+%Y%m%dT%H%M%SZ"], capture_output=True, text=True, check=True, ).stdout.strip() return f"{env_name}-{stamp}" def write_deploy_summary(summary_path: Path, payload: dict[str, str]) -> None: summary_path.parent.mkdir(parents=True, exist_ok=True) lines = [f"{key}={value}" for key, value in payload.items()] summary_path.write_text("\n".join(lines) + "\n") def validate_prod_tidb_binding(updates: dict[str, str]) -> None: expected = { "TIDB_PORT": EXPECTED_PROD_TIDB_PORT, "TIDB_DATABASE": EXPECTED_PROD_TIDB_DATABASE, } mismatches = [ f"{key}={updates.get(key, '')!r} expected {value!r}" for key, value in expected.items() if str(updates.get(key, "")).strip() != value ] forbidden = non_prod_tidb_bindings() tidb_host = str(updates.get("TIDB_HOST", "")).strip() tidb_user = str(updates.get("TIDB_USER", "")).strip() if not tidb_host: mismatches.append("TIDB_HOST is empty") if not tidb_user: mismatches.append("TIDB_USER is empty") if tidb_user.lower() == "root" or tidb_user.lower().endswith(".root"): mismatches.append("TIDB_USER must be the least-privilege Prod runtime identity, not root") if tidb_user and tidb_user in forbidden["users"]: mismatches.append(f"TIDB_USER matches non-prod target {tidb_user!r}") if mismatches: raise common.DeployError("invalid multi_prod TiDB binding: " + "; ".join(mismatches)) def non_prod_tidb_bindings() -> dict[str, set[str]]: config = json.loads(DEPLOYMENT_CONFIG.read_text()) users: set[str] = set() branch_ids: set[str] = set() for env_name, env_config in config.get("environments", {}).items(): if env_name == "multi_prod": continue topology = env_config.get("topology", {}) for user in topology.get("additional_non_prod_tidb_users", []): user = str(user).strip() if user: users.add(user) for branch_id in topology.get("additional_non_prod_tidb_branch_ids", []): branch_id = str(branch_id).strip() if branch_id: branch_ids.add(branch_id) target_config = topology.get("target_config", {}) for target in target_config.values(): tidb = target.get("tidb", {}) user = str(tidb.get("user", "")).strip() branch_id = str(tidb.get("branch_id", "")).strip() if user: users.add(user) if branch_id: branch_ids.add(branch_id) return {"users": users, "branch_ids": branch_ids} PROD_HUBSPOT_RUNTIME_ALIAS_MAP = { "APP_NAME": "EXP_HUBSPOT_APP_NAME", "APP_ID": "EXP_HUBSPOT_APP_ID", "CLIENT_ID": "EXP_HUBSPOT_CLIENT_ID", "AUTH_TYPE": "EXP_HUBSPOT_AUTH_TYPE", "WEBHOOK_URL": "EXP_HUBSPOT_WEBHOOK_URL", "CLIENT_SECRET": "EXP_HUBSPOT_CLIENT_SECRET", "WORKFLOW_ACTION_SIGNING_SECRET": "EXP_HUBSPOT_WORKFLOW_ACTION_SIGNING_SECRET", "ACCESS_TOKEN": "EXP_HUBSPOT_ACCESS_TOKEN", "ACTION_OPTIONS_TOKEN": "EXP_HUBSPOT_ACTION_OPTIONS_TOKEN", "WORKFLOW_ACTION_ID": "EXP_HUBSPOT_WORKFLOW_ACTION_ID", "EXTERNAL_ADAPTER_BASE_URL": "EXP_HUBSPOT_EXTERNAL_ADAPTER_BASE_URL", } def hubspot_prod_runtime_alias_updates() -> dict[str, str]: updates: dict[str, str] = {} for suffix, exp_key in PROD_HUBSPOT_RUNTIME_ALIAS_MAP.items(): prod_key = f"PROD_HUBSPOT_{suffix}" value = os.environ.get(prod_key, "").strip() or os.environ.get(exp_key, "").strip() updates[prod_key] = value updates[exp_key] = value return updates def prod_smtp_runtime_updates() -> dict[str, str]: enabled = os.environ.get("PROD_SMTP_ENABLED", "false").strip().lower() if enabled in {"", "false"}: return {} if enabled != "true": raise common.DeployError("PROD_SMTP_ENABLED must be true or false") updates = { "SMTP_HOST": os.environ.get("PROD_SMTP_HOST", "").strip(), "SMTP_PORT": os.environ.get("PROD_SMTP_PORT", "").strip(), "SMTP_USER": os.environ.get("PROD_SMTP_USER", "").strip(), "SMTP_PASSWORD": os.environ.get("PROD_SMTP_PASSWORD", "").strip(), "SMTP_FROM": os.environ.get("PROD_SMTP_FROM", "").strip(), "SMTP_TLS": os.environ.get("PROD_SMTP_TLS", "").strip().lower(), } missing = [key for key, value in updates.items() if not value] if missing: raise common.DeployError( "missing required prod SMTP settings while PROD_SMTP_ENABLED=true: " + ", ".join(missing) ) expected = { "SMTP_HOST": EXPECTED_PROD_SMTP_HOST, "SMTP_PORT": EXPECTED_PROD_SMTP_PORT, "SMTP_FROM": EXPECTED_PROD_SMTP_FROM, "SMTP_TLS": "true", } mismatches = [ f"{key}={updates[key]!r} expected {value!r}" for key, value in expected.items() if updates[key] != value ] if mismatches: raise common.DeployError("invalid prod SES SMTP settings: " + "; ".join(mismatches)) return updates def fetch_release_tag(repo_root: Path, release_tag: str, gh_token: str) -> None: probe = subprocess.run( ["git", "-C", str(repo_root), "show-ref", "--verify", "--quiet", f"refs/tags/{release_tag}"], text=True, capture_output=True, check=False, ) if probe.returncode == 0: return token = gh_token.strip() if not token: raise common.DeployError(f"missing GH_TOKEN to fetch release tag {release_tag}") fetch_env = dict(os.environ) fetch_env.update(common.private_go_module_env({"GH_TOKEN": token})) fetch = subprocess.run( ["git", "-C", str(repo_root), "fetch", "--depth=1", "origin", f"refs/tags/{release_tag}:refs/tags/{release_tag}"], text=True, capture_output=True, check=False, env=fetch_env, ) if fetch.returncode != 0: raise common.DeployError(f"failed to fetch release tag {release_tag} in {repo_root}") def release_tag_target_commit(repo_root: Path, release_tag: str) -> str: return common.release_tag_target_commit(repo_root, release_tag) def require_payload_field(payload: dict[str, object], key: str) -> str: value = str(payload.get(key, "")).strip() if not value: raise common.DeployError(f"release tag payload missing {key}") return value def git_show_json(repo_root: Path, commit: str, path: str) -> dict[str, object]: raw = common.run(["git", "-C", str(repo_root), "show", f"{commit}:{path}"], capture=True) try: payload = json.loads(raw) except json.JSONDecodeError as exc: raise common.DeployError(f"{path} in tagged commit is not valid JSON: {exc}") from exc return common.require_json_object(payload, f"{commit}:{path}") def final_pack_lineage_refs_from_commit(repo_root: Path, commit: str) -> dict[str, str]: if not common.GIT_COMMIT_RE.fullmatch(commit): raise common.DeployError(f"release tag target commit is not a 40-character lowercase SHA: {commit!r}") stack_lock = git_show_json(repo_root, commit, "stack.lock.json") pack_build = common.require_json_object(stack_lock.get("pack_build", {}), "stack.lock.json pack_build") final_pack_path = str(pack_build.get("final_pack_path") or "packs/NOT-EDIT-final-merged-packs").strip() if not final_pack_path: raise common.DeployError("stack.lock.json pack_build.final_pack_path is empty in tagged commit") lineage_path = f"{final_pack_path}/.build-lineage.json" lineage = git_show_json(repo_root, commit, lineage_path) upstream = common.require_json_object(lineage.get("upstreamRuntime", {}), f"{lineage_path} upstreamRuntime") official_pack = common.require_json_object(lineage.get("officialPack", {}), f"{lineage_path} officialPack") upstream_ref = str(upstream.get("ref", "")).strip() official_app_packs_ref = str(official_pack.get("ref", "")).strip() missing = [] if not upstream_ref: missing.append("upstreamRuntime.ref") if not official_app_packs_ref: missing.append("officialPack.ref") if missing: raise common.DeployError(f"final pack build lineage missing required refs: {', '.join(missing)}") return { "upstream_ref": upstream_ref, "official_app_packs_ref": official_app_packs_ref, "lineage_path": f"{commit}:{lineage_path}", } def require_release_candidate_executor(raw_payload: dict[str, object], target_commit: str) -> str: contract_version = str(raw_payload.get("contract_version", "")).strip() if contract_version != RELEASE_CANDIDATE_CONTRACT_VERSION: raise common.DeployError( "release tag has unsupported candidate contract " f"for prod promotion: {contract_version!r}" ) if raw_payload.get("prod_release_eligible") is not True: raise common.DeployError("release tag is not prod-release eligible") setup_apply = raw_payload.get("setup_apply") if not isinstance(setup_apply, dict): raise common.DeployError("release tag is missing staging setup-apply evidence") try: require_prod_release_setup_evidence(setup_apply) except ValueError as exc: raise common.DeployError(f"release tag staging setup-apply evidence is not prod parity: {exc}") from exc try: require_prod_promotion_contract( raw_payload.get("ordinary_prod_promotion"), expected_final_pack_hash=raw_payload.get("final_pack_hash"), expected_official_app_packs_commit=raw_payload.get( "official_app_packs_commit" ), ) except ValueError as exc: raise common.DeployError( f"release tag ordinary Prod promotion contract is invalid: {exc}" ) from exc payload_target_commit = require_payload_field(raw_payload, "tag_target_commit") if payload_target_commit != target_commit: raise common.DeployError( "release tag payload target commit does not match tagged commit " f"(payload {payload_target_commit}, tag target {target_commit})" ) payload_downstream_commit = require_payload_field(raw_payload, "downstream_commit") if payload_downstream_commit != target_commit: raise common.DeployError( "release tag payload downstream commit does not match tagged commit " f"(payload {payload_downstream_commit}, tag target {target_commit})" ) executor = raw_payload.get("executor") if not isinstance(executor, dict): raise common.DeployError("release tag is missing candidate executor") if str(executor.get("kind", "")).strip() != "downstream_git_commit": raise common.DeployError("release tag candidate executor kind is invalid") if not str(executor.get("repository", "")).strip(): raise common.DeployError("release tag candidate executor repository is missing") executor_commit = str(executor.get("commit", "")).strip() if executor_commit != target_commit: raise common.DeployError( "release tag candidate executor does not match tagged commit " f"(executor {executor_commit}, tag target {target_commit})" ) return executor_commit def require_candidate_bundle( raw_payload: dict[str, object], *, target_commit: str, upstream_ref: str, official_app_packs_ref: str, final_pack_hash: str, ) -> dict[str, str]: bundle = raw_payload.get("candidate_bundle") if not isinstance(bundle, dict): raise common.DeployError("release tag is missing candidate bundle") source_run_id = str(raw_payload.get("source_run_id", "")).strip() expected = { "downstream_ref": target_commit, "upstream_ref": upstream_ref, "official_app_packs_ref": official_app_packs_ref, "final_pack_hash": final_pack_hash, "source_run_id": source_run_id, "artifact_name": f"staging-release-manifest-{source_run_id}", } if not source_run_id.isdigit(): raise common.DeployError("release tag candidate bundle source_run_id is invalid") if bundle.get("contract_version") != BUNDLE_CONTRACT_VERSION: raise common.DeployError("release tag candidate bundle contract_version is invalid") archive_relative_path = str( bundle.get("archive_relative_path", "") ).strip() if archive_relative_path != ( "deployment-bundle/deployment-bundle.tar" ): raise common.DeployError( "release tag candidate bundle archive_relative_path is invalid" ) if str(bundle.get("manifest_name", "")).strip() != "deployment-bundle.json": raise common.DeployError("release tag candidate bundle manifest_name is invalid") archive_sha256 = str(bundle.get("archive_sha256", "")).strip() manifest_sha256 = str(bundle.get("manifest_sha256", "")).strip() connect_mcp_url = str(bundle.get("connect_mcp_url", "")).strip() for key, value in ( ("archive_sha256", archive_sha256), ("manifest_sha256", manifest_sha256), ): if len(value) != 64 or any(char not in "0123456789abcdef" for char in value): raise common.DeployError(f"release tag candidate bundle {key} is invalid") for key, expected_value in expected.items(): if str(bundle.get(key, "")).strip() != expected_value: raise common.DeployError( f"release tag candidate bundle {key} does not match release identity" ) if connect_mcp_url != "https://nexuscrm-new-prod.pingcap.com/mcp": raise common.DeployError( "release tag candidate bundle connect_mcp_url is invalid" ) return { "candidate_bundle_artifact_name": expected["artifact_name"], "candidate_bundle_archive_relative_path": archive_relative_path, "candidate_bundle_archive_sha256": archive_sha256, "candidate_bundle_manifest_sha256": manifest_sha256, "candidate_connect_mcp_url": connect_mcp_url, } def resolve_release_tag_config( release_tag: str, repo_root: Path, ) -> dict[str, object]: fetch_release_tag(repo_root, release_tag, os.environ.get("GH_TOKEN", "")) raw_payload = common.load_release_tag_payload(repo_root, release_tag) kind = str(raw_payload.get("kind", "")).strip() if kind != "staging_release_tag": raise common.DeployError(f"unexpected release tag kind for {release_tag}: {kind!r}") payload_tag_name = str(raw_payload.get("tag_name", "")).strip() if payload_tag_name != release_tag: raise common.DeployError( "release tag payload name does not match selected tag " f"(payload {payload_tag_name!r}, selected {release_tag!r})" ) environment = str(raw_payload.get("environment", "")).strip() if environment != "staging": raise common.DeployError(f"release tag environment mismatch for {release_tag}: {environment!r}") candidate_reset_branch = raw_payload.get("reset_branch") if not isinstance(candidate_reset_branch, bool): raise common.DeployError( f"release tag {release_tag} is missing boolean reset_branch evidence" ) target_commit = release_tag_target_commit(repo_root, release_tag) executor_commit = require_release_candidate_executor(raw_payload, target_commit) embedded_ordinary_prod_promotion = require_prod_promotion_contract( raw_payload.get("ordinary_prod_promotion"), expected_final_pack_hash=raw_payload.get("final_pack_hash"), expected_official_app_packs_commit=raw_payload.get( "official_app_packs_commit" ), ) try: exact_ordinary_prod_promotion = build_prod_promotion_contract_from_git( repo_root, revision=target_commit, final_pack_hash=require_payload_field( raw_payload, "final_pack_hash", ), final_pack_name=require_payload_field( raw_payload, "final_pack_pack_name", ), final_pack_version=require_payload_field( raw_payload, "final_pack_pack_version", ), official_app_packs_commit=require_payload_field( raw_payload, "official_app_packs_commit", ), ) except (subprocess.CalledProcessError, UnicodeDecodeError, ValueError) as exc: raise common.DeployError( "release tag ordinary Prod promotion contract could not be " f"derived from the exact tag tree: {exc}" ) from exc if embedded_ordinary_prod_promotion != exact_ordinary_prod_promotion: raise common.DeployError( "release tag ordinary Prod promotion contract does not match " "the exact candidate tag tree" ) ordinary_prod_promotion = exact_ordinary_prod_promotion app_shell = raw_payload.get("app_shell") if not isinstance(app_shell, dict) or str(app_shell.get("status", "")).strip() != "passed": raise common.DeployError(f"release tag {release_tag} is missing passed app-shell evidence") app_shell_config_hash = str(app_shell.get("config_sha256", "")).strip() if len(app_shell_config_hash) != 64 or any(ch not in "0123456789abcdef" for ch in app_shell_config_hash): raise common.DeployError(f"release tag {release_tag} has invalid app-shell config hash") lineage_refs = final_pack_lineage_refs_from_commit(repo_root, target_commit) final_pack_hash = require_payload_field(raw_payload, "final_pack_hash") candidate_bundle = require_candidate_bundle( raw_payload, target_commit=target_commit, upstream_ref=lineage_refs["upstream_ref"], official_app_packs_ref=lineage_refs["official_app_packs_ref"], final_pack_hash=final_pack_hash, ) try: _, promotion_authority_sha256 = require_tag_promotion_authority(raw_payload) tag_bound_authority = build_promotion_authority( release_tag=release_tag, tag_target_commit=target_commit, executor_commit=executor_commit, upstream_ref=lineage_refs["upstream_ref"], official_app_packs_ref=lineage_refs["official_app_packs_ref"], final_pack_hash=require_payload_field(raw_payload, "final_pack_hash"), ) if tag_bound_authority != raw_payload.get("promotion_authority"): raise ValueError("release tag promotion authority does not match final-pack lineage") except ValueError as exc: raise common.DeployError(f"release tag promotion authority is invalid: {exc}") from exc return { "release_tag": release_tag, "authority": "final_pack_lineage", "release_refs_source": "final_pack_lineage", "tag_target_commit": target_commit, "executor_commit": executor_commit, "downstream_ref": target_commit, "upstream_ref": lineage_refs["upstream_ref"], "official_app_packs_ref": lineage_refs["official_app_packs_ref"], "lineage_path": lineage_refs["lineage_path"], "source_run_id": str(raw_payload.get("source_run_id", "")).strip(), "candidate_reset_branch": ( "true" if candidate_reset_branch else "false" ), "final_pack_hash": final_pack_hash, "promotion_authority_sha256": promotion_authority_sha256, "app_shell_config_hash": app_shell_config_hash, "ordinary_prod_promotion_contract_sha256": str( ordinary_prod_promotion["contract_sha256"] ), "ordinary_prod_shared_state_digest": str( ordinary_prod_promotion["shared_state_digest"] ), "ordinary_prod_promotion_enforcement_status": str( ordinary_prod_promotion["enforcement_status"] ), "ordinary_prod_promotion": ordinary_prod_promotion, **candidate_bundle, } def validate_retained_runtime_env_contract( *, release_tag: str, repo_root: Path, runtime_env_path: Path, expected_executor_commit: str, ) -> dict[str, str]: if not runtime_env_path.is_file(): raise common.DeployError( f"cannot resume: retained runtime env is missing: {runtime_env_path}" ) if not common.GIT_COMMIT_RE.fullmatch(expected_executor_commit): raise common.DeployError( "cannot resume: expected executor commit is not a 40-character " "lowercase SHA" ) release_config = resolve_release_tag_config(release_tag, repo_root) if release_config["executor_commit"] != expected_executor_commit: raise common.DeployError( "cannot resume: selected release contract executor does not match " "the deployment authority" ) retained = common.load_env_file(runtime_env_path) mismatched_keys = sorted( env_key for env_key, config_key in RETAINED_RELEASE_CONFIG_BINDINGS.items() if str(retained.get(env_key, "")).strip() != str(release_config[config_key]).strip() ) if mismatched_keys: raise common.DeployError( "cannot resume: retained runtime env does not match the validated " "release contract for keys: " + ", ".join(mismatched_keys) ) return release_config def refresh_retained_runtime_secret_alias(runtime_env_path: Path) -> None: """Refresh the canonical adapter password from the existing Prod secret.""" admin_password = os.environ.get("NEXUS_ADMIN_ONLINE_PASSWORD", "").strip() if not admin_password: raise common.DeployError( "cannot resume: missing NEXUS_ADMIN_ONLINE_PASSWORD" ) update_env_file( runtime_env_path, { "NEXUS_ADMIN_PASSWORD": admin_password, "PYTHONDONTWRITEBYTECODE": "1", }, ) def validate_retained_ordinary_prod_admission( *, runtime_env_path: Path, release_config: dict[str, object], ) -> dict[str, object]: """Validate and preserve the original transition receipt for resume. The baseline is the source release that setup migrated from. After cutover the live release is the candidate, so rebuilding this receipt would erase the original mismatch/downtime fact and make a valid setup receipt unverifiable. Live marker, slot, and source-tree state are rechecked by the resume executor before it accepts this persisted transition. """ retained = common.load_env_file(runtime_env_path) version_root = Path( str(retained.get("ENV_ROOT", "")).strip() ).expanduser().resolve() deploy_id = str(retained.get("DEPLOY_ID", "")).strip() artifact_dir = Path( str(retained.get("DEPLOY_ARTIFACT_DIR", "")).strip() ).expanduser().resolve() expected_artifact_dir = ( version_root / "artifacts" / "deploys" / deploy_id ).resolve() if ( not str(retained.get("MULTI_PROD_ROOT", "")).strip() or not str(retained.get("ENV_ROOT", "")).strip() or not deploy_id or not str(retained.get("DEPLOY_ARTIFACT_DIR", "")).strip() or artifact_dir != expected_artifact_dir ): raise common.DeployError( "cannot resume: retained runtime baseline evidence path is invalid" ) backend_port_slots = tuple( value.strip() for value in str(retained.get("BACKEND_PORT_SLOTS", "")).split(",") if value.strip() ) if ( len(backend_port_slots) < 2 or any(not value.isdigit() for value in backend_port_slots) ): raise common.DeployError( "cannot resume: retained runtime backend port slots are invalid" ) receipt_path = artifact_dir / "ordinary-prod-baseline-comparison.json" if receipt_path.is_symlink() or not receipt_path.is_file(): raise common.DeployError( "cannot resume: original Prod transition receipt is missing or invalid" ) try: comparison = require_prod_baseline_receipt( json.loads(receipt_path.read_text()), catalog_migration=True, ) candidate_contract = require_prod_promotion_contract( release_config["ordinary_prod_promotion"] ) except (OSError, json.JSONDecodeError, ValueError) as exc: raise common.DeployError( f"cannot resume: original Prod transition receipt is invalid: {exc}" ) from exc candidate = comparison["candidate"] expected_candidate = { "release_tag": str(release_config["release_tag"]), "tag_target_commit": str(release_config["tag_target_commit"]), "contract_sha256": str(candidate_contract["contract_sha256"]), "shared_state_digest": str(candidate_contract["shared_state_digest"]), } mismatched_candidate = sorted( key for key, expected in expected_candidate.items() if str(candidate.get(key, "")).strip() != expected ) expected_runtime = { "PROD_RELEASE_MODE": "metadata-first-maintenance", "PROD_ORDINARY_ACTIVE_BASELINE_STATUS": str(comparison["status"]), "PROD_ORDINARY_ACTIVE_BASELINE_MATCH": ( "true" if comparison["baseline_match"] else "false" ), "PROD_ORDINARY_ACTIVE_BASELINE_RECEIPT_SHA256": str( comparison["receipt_sha256"] ), "PROD_ORDINARY_MIGRATED_BASELINE_STATUS": "pending", "PROD_ORDINARY_MIGRATED_BASELINE_RECEIPT_SHA256": "", } mismatched_runtime = sorted( key for key, expected in expected_runtime.items() if str(retained.get(key, "")).strip() != expected ) if mismatched_candidate or mismatched_runtime: raise common.DeployError( "cannot resume: original Prod transition receipt does not match " "the retained candidate: " + ", ".join(mismatched_candidate + mismatched_runtime) ) update_env_file( runtime_env_path, {}, remove_keys=set(RETIRED_PROD_CONTROL_PLANE_RUNTIME_KEYS), ) return comparison def compare_active_ordinary_prod_baseline( repo_root: Path, *, active_release_tag: str, candidate_release_tag: str, candidate_tag_target_commit: str, candidate_contract: object, staging_proved_l2_migration: bool = False, ) -> dict[str, object]: normalized_active_tag = active_release_tag.strip() if not normalized_active_tag or normalized_active_tag == PROD_RESET_NO_ACTIVE_RELEASE: return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag="", active_tag_target_commit="", active_contract=None, active_contract_status="no-active-release", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) try: fetch_release_tag( repo_root, normalized_active_tag, os.environ.get("GH_TOKEN", ""), ) active_payload = common.load_release_tag_payload( repo_root, normalized_active_tag, ) except (common.DeployError, OSError, ValueError): return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit="", active_contract=None, active_contract_status="unavailable-active-contract", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) if str(active_payload.get("tag_name", "")).strip() != normalized_active_tag: return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit="", active_contract=None, active_contract_status="invalid-active-contract", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) active_final_pack_hash = str( active_payload.get("final_pack_hash", "") ).strip() active_official_app_packs_commit = str( active_payload.get("official_app_packs_commit", "") ).strip() active_final_pack_name = str( active_payload.get("final_pack_pack_name", "") ).strip() active_final_pack_version = str( active_payload.get("final_pack_pack_version", "") ).strip() active_target_commit = str( active_payload.get("tag_target_commit", "") ).strip() if ( not active_final_pack_hash or not active_official_app_packs_commit or not active_final_pack_name or not active_final_pack_version or not common.GIT_COMMIT_RE.fullmatch(active_target_commit) ): return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit="", active_contract=None, active_contract_status="invalid-active-contract", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) try: actual_active_target = subprocess.run( [ "git", "-C", str(repo_root), "rev-parse", "--verify", f"refs/tags/{normalized_active_tag}^{{commit}}", ], check=True, capture_output=True, text=True, ).stdout.strip() except subprocess.CalledProcessError: actual_active_target = "" if actual_active_target != active_target_commit: return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit="", active_contract=None, active_contract_status="invalid-active-contract", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) try: derived_active_contract = build_prod_promotion_contract_from_git( repo_root, revision=active_target_commit, final_pack_hash=active_final_pack_hash, final_pack_name=active_final_pack_name, final_pack_version=active_final_pack_version, official_app_packs_commit=active_official_app_packs_commit, ) return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit=active_target_commit, active_contract=derived_active_contract, active_contract_source="derived-exact-active-tag", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) except (subprocess.CalledProcessError, UnicodeDecodeError, ValueError): return build_active_baseline_comparison( candidate_contract, candidate_release_tag=candidate_release_tag, candidate_tag_target_commit=candidate_tag_target_commit, active_release_tag=normalized_active_tag, active_tag_target_commit="", active_contract=None, active_contract_status="invalid-active-contract", catalog_migration=True, staging_proved_l2_migration=staging_proved_l2_migration, ) def materialize_migrated_prod_baseline_receipt( *, multi_prod_root: Path, artifact_dir: Path, baseline_transition: dict[str, object], expected_final_pack_hash: str, ) -> dict[str, object] | None: """Copy one exact, host-authorized migration receipt into this deploy.""" if bool(baseline_transition["baseline_match"]): return None candidate = baseline_transition["candidate"] if not isinstance(candidate, dict): raise common.DeployError( "ordinary Prod baseline candidate identity is invalid" ) candidate_release_tag = str(candidate["release_tag"]).strip() authority_path = ( multi_prod_root / "shared" / "prod-baseline-migrations" / f"{candidate_release_tag}.json" ) if authority_path.is_symlink() or not authority_path.is_file(): raise common.DeployError( "ordinary Prod shared-state mismatch has no exact migrated-baseline " f"authority for {candidate_release_tag}" ) try: receipt = require_bound_migrated_baseline_receipt( json.loads(authority_path.read_text()), baseline_transition=baseline_transition, expected_final_pack_hash=expected_final_pack_hash, ) except (OSError, json.JSONDecodeError, ValueError) as exc: raise common.DeployError( f"ordinary Prod migrated-baseline authority is invalid: {exc}" ) from exc deploy_receipt_path = artifact_dir / "ordinary-prod-migrated-baseline.json" deploy_receipt_path.write_text( json.dumps(receipt, indent=2, sort_keys=True) + "\n" ) return receipt def prepare_runtime_env(args: argparse.Namespace) -> Path: connected_mail_oauth = connected_mail_oauth_env( os.environ, error_type=common.DeployError, ) outer_env_root = runtime_env_file(args.env).parents[1] outer_env_root.mkdir(parents=True, exist_ok=True) configured_backend_slots = tuple( value.strip() for value in os.environ.get( "MULTI_PROD_BACKEND_PORT_SLOTS", ",".join(DEFAULT_PROD_BACKEND_PORT_SLOTS), ).split(",") if value.strip() ) if len(configured_backend_slots) < 2 or any(not value.isdigit() for value in configured_backend_slots): raise common.DeployError("MULTI_PROD_BACKEND_PORT_SLOTS must contain at least two numeric ports") active_runtime_identity = capture_prod_reset_source_identity( outer_env_root, backend_port_slots=configured_backend_slots, ) active_release_tag = active_runtime_identity["release_tag"] workspace_repo_root = Path(os.environ.get("GITHUB_WORKSPACE", Path.cwd())).expanduser().resolve() release_config = resolve_release_tag_config( args.release_tag, workspace_repo_root, ) expected_connect_mcp_url = f"{args.public_base_url.rstrip('/')}/mcp" if release_config["candidate_connect_mcp_url"] != expected_connect_mcp_url: raise common.DeployError( "selected release candidate Connect URL does not match Prod public base URL" ) version_root = release_root(args.env, release_config["release_tag"]) env_file = release_runtime_env_file(args.env, release_config["release_tag"]) version_root.mkdir(parents=True, exist_ok=True) env_file.parent.mkdir(parents=True, exist_ok=True) deploy_id = build_deploy_id(args.env) deploy_artifact_dir = version_root / "artifacts" / "deploys" / deploy_id deploy_artifact_dir.mkdir(parents=True, exist_ok=True) active_baseline_comparison = compare_active_ordinary_prod_baseline( workspace_repo_root, active_release_tag=active_release_tag, candidate_release_tag=str(release_config["release_tag"]), candidate_tag_target_commit=str( release_config["tag_target_commit"] ), candidate_contract=release_config["ordinary_prod_promotion"], staging_proved_l2_migration=( release_config["candidate_reset_branch"] == "true" ), ) active_baseline_comparison_path = ( deploy_artifact_dir / "ordinary-prod-baseline-comparison.json" ) active_baseline_comparison_path.write_text( json.dumps(active_baseline_comparison, indent=2, sort_keys=True) + "\n" ) try: require_prod_baseline_receipt( active_baseline_comparison, catalog_migration=True, ) except ValueError as exc: raise common.DeployError( "Prod transition admission blocked before production mutation: " f"{active_baseline_comparison['status']}; {exc}" ) from exc release_mode = "metadata-first-maintenance" (outer_env_root / "current" / "env").mkdir(parents=True, exist_ok=True) (version_root / "repos").mkdir(parents=True, exist_ok=True) (version_root / ".runtime").mkdir(parents=True, exist_ok=True) (version_root / "logs").mkdir(parents=True, exist_ok=True) (version_root / "run").mkdir(parents=True, exist_ok=True) (version_root / "frontend-dist-build").mkdir(parents=True, exist_ok=True) hubspot_aliases = hubspot_prod_runtime_alias_updates() workflow_action_id = hubspot_aliases["EXP_HUBSPOT_WORKFLOW_ACTION_ID"] if not workflow_action_id: raise common.DeployError("missing PROD_HUBSPOT_WORKFLOW_ACTION_ID") smtp_updates = prod_smtp_runtime_updates() salesforce_file_storage_bucket = str( os.environ.get( "PROD_SALESFORCE_FILE_STORAGE_BUCKET", DEFAULT_PROD_SALESFORCE_FILE_STORAGE_BUCKET, ) or "" ).strip() salesforce_file_storage_region = str( os.environ.get( "PROD_SALESFORCE_FILE_STORAGE_REGION", DEFAULT_PROD_SALESFORCE_FILE_STORAGE_REGION, ) or "" ).strip() if not salesforce_file_storage_bucket or not salesforce_file_storage_region: raise common.DeployError( "Prod Salesforce file-storage bucket and region are required" ) require_prod_salesforce_file_storage_iam_authority( bucket=salesforce_file_storage_bucket, region=salesforce_file_storage_region, ) additional_file_storage_authorities = [ { "name": "salesforce-history", "source_type": "salesforce", "provider": "s3", "bucket": salesforce_file_storage_bucket, "region": salesforce_file_storage_region, "key_prefix": PROD_SALESFORCE_FILE_STORAGE_PREFIX, } ] updates = { "ENV_NAME": "multi-prod", "ENV_ROOT": str(version_root), "MULTI_PROD_ROOT": str(outer_env_root), "DEPLOY_MODE": "release", "DEPLOYMENT_AUTHORITY_MODE": release_config["authority"], "RELEASE_REFS_SOURCE": release_config["release_refs_source"], "DEPLOY_ID": deploy_id, "DEPLOY_ARTIFACT_DIR": str(deploy_artifact_dir), "DOWNSTREAM_REPO_URL": "https://github.com/pingcap-inc/nexuscrm-pingcap-pack.git", "UPSTREAM_REPO_URL": "https://github.com/pingcap-inc/NexusCRM.git", "OFFICIAL_APP_PACKS_REPO_URL": "https://github.com/pingcap-inc/nexuscrm-app-packs.git", "DOWNSTREAM_REF": release_config["downstream_ref"], "UPSTREAM_REF": release_config["upstream_ref"], "OFFICIAL_APP_PACKS_REF": release_config["official_app_packs_ref"], "RELEASE_TAG": release_config["release_tag"], "TAG_TARGET_COMMIT": release_config["tag_target_commit"], "RELEASE_EXECUTOR_COMMIT": release_config["executor_commit"], "LINEAGE_PATH": release_config["lineage_path"], "STAGING_RELEASE_RUN_ID": release_config["source_run_id"], "PROD_CANDIDATE_RESET_BRANCH": release_config[ "candidate_reset_branch" ], "PROD_EXPECTED_FINAL_PACK_HASH": release_config["final_pack_hash"], "PROD_CANDIDATE_BUNDLE_ARTIFACT_NAME": release_config[ "candidate_bundle_artifact_name" ], "PROD_CANDIDATE_BUNDLE_ARCHIVE_RELATIVE_PATH": release_config[ "candidate_bundle_archive_relative_path" ], "PROD_CANDIDATE_BUNDLE_ARCHIVE_SHA256": release_config[ "candidate_bundle_archive_sha256" ], "PROD_CANDIDATE_BUNDLE_MANIFEST_SHA256": release_config[ "candidate_bundle_manifest_sha256" ], "PROD_CANDIDATE_CONNECT_MCP_URL": release_config[ "candidate_connect_mcp_url" ], "PROD_CANDIDATE_BUNDLE_DOWNLOAD_ROOT": str( version_root / "artifacts" / "staging-candidate" ), "PROD_CANDIDATE_BUNDLE_ARCHIVE_PATH": str( version_root / "artifacts" / "staging-candidate" / release_config["candidate_bundle_archive_relative_path"] ), "PROD_CANDIDATE_BUNDLE_MATERIALIZED_ROOT": str( version_root / "deployment-bundle" ), "PROD_PROMOTION_AUTHORITY_SHA256": release_config["promotion_authority_sha256"], "PROD_EXPECTED_APP_SHELL_CONFIG_SHA256": release_config["app_shell_config_hash"], "PROD_ORDINARY_PROMOTION_CONTRACT_SHA256": release_config[ "ordinary_prod_promotion_contract_sha256" ], "PROD_ORDINARY_SHARED_STATE_DIGEST": release_config[ "ordinary_prod_shared_state_digest" ], "PROD_ORDINARY_PROMOTION_ENFORCEMENT_STATUS": release_config[ "ordinary_prod_promotion_enforcement_status" ], "PROD_RELEASE_MODE": release_mode, "PROD_ORDINARY_ACTIVE_BASELINE_STATUS": str( active_baseline_comparison["status"] ), "PROD_ORDINARY_ACTIVE_BASELINE_MATCH": ( "true" if active_baseline_comparison["baseline_match"] else "false" ), "PROD_ORDINARY_ACTIVE_BASELINE_RECEIPT_SHA256": str( active_baseline_comparison["receipt_sha256"] ), "PROD_ORDINARY_MIGRATED_BASELINE_STATUS": "pending", "PROD_ORDINARY_MIGRATED_BASELINE_RECEIPT_SHA256": "", "BACKEND_PORT": "3104", "BACKEND_PORT_SLOTS": os.environ.get("MULTI_PROD_BACKEND_PORT_SLOTS", "").strip() or "3103,3104,3105,3106", "BACKEND_GO_BUILD_TAGS": "crm", "PUBLIC_BASE_URL": args.public_base_url, "SERVER_NAME": args.healthcheck_host, "HEALTHCHECK_HOST": args.healthcheck_host, "NGINX_CONF_NAME": "prod", "HUBSPOT_ORDINARY_WEBHOOK_MODE": os.environ.get( "HUBSPOT_ORDINARY_WEBHOOK_MODE", "proxy" ).strip() or "proxy", "HUBSPOT_WORKFLOW_EXECUTE_MODE": os.environ.get( "HUBSPOT_WORKFLOW_EXECUTE_MODE", "proxy" ).strip() or "proxy", "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS": ( prod_connector_webhook_ack_drop_endpoints( os.environ.get("NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS") ) ), "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS": ( prod_connector_webhook_ack_drop_timeout_ms( os.environ.get("NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS") ) ), "NGINX_LISTEN_PORT": "80", "NGINX_LISTEN_SUFFIX": "default_server", "NGINX_FORWARDED_PROTO": "https", "SYSTEMD_USER": "ubuntu", "TIDB_HOST": os.environ.get("MULTI_PROD_TIDB_HOST", "").strip(), "TIDB_PORT": os.environ.get("MULTI_PROD_TIDB_PORT", "").strip(), "TIDB_USER": os.environ.get("MULTI_PROD_TIDB_USER", "").strip(), "TIDB_PASSWORD": os.environ.get("MULTI_PROD_TIDB_PASSWORD", "").strip(), "TIDB_DATABASE": os.environ.get("MULTI_PROD_TIDB_DATABASE", "").strip(), "TIDB_TLS": "true", "VERIFY_USERNAME": os.environ.get("MULTI_PROD_VERIFY_USERNAME", "admin@example.invalid").strip() or "admin@example.invalid", "VERIFY_LAYOUT_OBJECT": os.environ.get("MULTI_PROD_VERIFY_LAYOUT_OBJECT", "Lead").strip() or "Lead", "VERIFY_LAYOUT_MIN_SECTIONS": os.environ.get("MULTI_PROD_VERIFY_LAYOUT_MIN_SECTIONS", "8").strip() or "8", "RESET_DB": "0", "RESET_TIDB_BRANCH": "0", "INSTALL_SYSTEMD": "1", "INSTALL_NGINX": "1", "SKIP_BOOTSTRAP": "false", "SETUP_REPAIR_LEGACY_UNIQUE_DUPLICATES": "false", "NEXUSCRM_RUNTIME_STANDBY": "true", "PYTHONDONTWRITEBYTECODE": "1", "NEXUSCRM_PUBLIC_OAUTH_PKCE_ENABLED": "true", # App-blind L0 download endpoint (GET /downloads/{kind}/latest) reads its # artifact catalog from this env; the tenant supplies the concrete # nexuscrm-connect artifacts. Keys embed the agent-tools package version # (immutable per release); the release publish-artifact step uploads to # exactly these keys. Single-quoted because run_backend.sh sources this # env file with bash. "NEXUSCRM_DOWNLOAD_ARTIFACTS": ( "'" + common.connect_download_artifacts_env_value(common.connect_package_version(REPO_ROOT)) + "'" ), "JWT_SECRET": os.environ.get("JWT_SECRET", "").strip(), "NEXUSCRM_NC_MASTER_KEY": os.environ.get("NEXUSCRM_NC_MASTER_KEY", "").strip(), **connected_mail_oauth, "NEXUSCRM_SCIM_INPROCESS_ENABLED": scim_inprocess_enabled( os.environ.get("NEXUSCRM_SCIM_INPROCESS_ENABLED") ), "NEXUS_ADMIN_ONLINE_PASSWORD": os.environ.get("NEXUS_ADMIN_ONLINE_PASSWORD", "").strip(), "NEXUS_ADMIN_PASSWORD": os.environ.get("NEXUS_ADMIN_ONLINE_PASSWORD", "").strip(), "GH_TOKEN": os.environ.get("GH_TOKEN", "").strip(), "NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK": release_connector_inbound_workers_per_pack( os.environ.get("NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK", SAFE_CONNECTOR_INBOUND_WORKERS_PER_PACK) ), "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS": release_connector_inbound_max_processing_rows( os.environ.get( "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS", SAFE_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS, ) ), # This is the active policy. L0 global standby prevents every request # and background worker from starting before the cutover activation. "NEXUSCRM_CONNECTOR_WORKITEM_CLAIM_ENABLED": "true", "NEXUSCRM_CONNECTOR_LIVE_QUEUE_STATS_ENABLED": prod_connector_live_queue_stats_enabled( os.environ.get("NEXUSCRM_CONNECTOR_LIVE_QUEUE_STATS_ENABLED") ), "CONNECTOR_PUSH_ASYNC_ENABLED": "true", "CONNECTOR_PUSH_WORKER_ENABLED": "true", "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS": os.environ.get( "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS", "true" ).strip() or "true", "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_LIMIT": os.environ.get( "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_LIMIT", "8" ).strip() or "8", "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_LIMIT": os.environ.get( "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_LIMIT", "8" ).strip() or "8", "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_MS": os.environ.get( "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_MS", "500" ).strip() or "500", "NEXUSCRM_CONNECTOR_WEBHOOK_RETRY_AFTER_SECONDS": os.environ.get( "NEXUSCRM_CONNECTOR_WEBHOOK_RETRY_AFTER_SECONDS", "10" ).strip() or "10", "DEPLOY_MULTI_PROD_HUBSPOT_QUIESCE_RETRY_AFTER_SECONDS": os.environ.get( "DEPLOY_MULTI_PROD_HUBSPOT_QUIESCE_RETRY_AFTER_SECONDS", "120" ).strip() or "120", "DEPLOY_CONNECTOR_INGRESS_RETRY_AFTER_SECONDS": os.environ.get( "DEPLOY_CONNECTOR_INGRESS_RETRY_AFTER_SECONDS", os.environ.get("DEPLOY_MULTI_PROD_HUBSPOT_QUIESCE_RETRY_AFTER_SECONDS", "120"), ).strip() or "120", "RATE_LIMIT_RPS": os.environ.get("RATE_LIMIT_RPS", "500").strip() or "500", "RATE_LIMIT_BURST": os.environ.get("RATE_LIMIT_BURST", "1000").strip() or "1000", "GOOGLE_CLIENT_ID": os.environ.get("GOOGLE_CLIENT_ID", "").strip(), "GOOGLE_CLIENT_SECRET": os.environ.get("GOOGLE_CLIENT_SECRET", "").strip(), "REACT_APP_GOOGLE_CLIENT_ID": os.environ.get("REACT_APP_GOOGLE_CLIENT_ID", "").strip(), "NEXUS_LOGIN_METHODS": os.environ.get("NEXUS_LOGIN_METHODS", "").strip(), **hubspot_aliases, "EXP_HUBSPOT_WORKFLOW_ACTION_CALLBACK_ACCESS_TOKEN": hubspot_aliases[ "EXP_HUBSPOT_ACCESS_TOKEN" ], "EXP_HUBSPOT_WORKFLOW_ACTION_POLICIES_JSON": ( "'" + json.dumps( {workflow_action_id: PROD_HUBSPOT_WORKFLOW_ACTION_POLICY}, separators=(",", ":"), ) + "'" ), "EXP_HUBSPOT_WORKFLOW_ACTION_QUEUE_DB": str( outer_env_root / "shared" / "hubspot-workflow-action-queue.sqlite3" ), "EXP_HUBSPOT_WORKFLOW_ACTION_REPLAY_RPS": PROD_HUBSPOT_WORKFLOW_ACTION_REPLAY_RPS, "FILE_STORAGE_PROVIDER": "s3", "FILE_STORAGE_BUCKET": PROD_FILE_STORAGE_BUCKET, "FILE_STORAGE_REGION": PROD_FILE_STORAGE_REGION, "FILE_STORAGE_PREFIX": PROD_FILE_STORAGE_PREFIX, "FILE_STORAGE_ADDITIONAL_S3_AUTHORITIES_JSON": ( "'" + json.dumps( additional_file_storage_authorities, separators=(",", ":"), sort_keys=True, ) + "'" ), # Retain one release-independent filesystem authority for pre-S3 rows # and legacy uploads while new customer files use the S3 authority. "FILE_STORAGE_ROOT": str( (outer_env_root / "shared" / "data" / "object-storage").resolve() ), "FILE_STORAGE_LEGACY_UPLOAD_ROOT": str( (outer_env_root / "shared" / "data" / "uploads").resolve() ), # The Prod deploy resolves this candidate-relative path to an absolute # release root before starting the backend. It is intentionally distinct # from mutable customer file storage. "NEXUSCRM_DOWNLOAD_ARTIFACT_STORAGE_ROOT": "runtime-data/connect-artifacts", } updates["PROD_EXPECTED_CONNECTOR_INBOUND_WORKERS_PER_PACK"] = updates[ "NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK" ] updates["PROD_EXPECTED_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS"] = updates[ "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS" ] updates.update(smtp_updates) common.hubspot_ingress_nginx_modes(updates) required_keys = [ "TIDB_HOST", "TIDB_PORT", "TIDB_USER", "TIDB_PASSWORD", "TIDB_DATABASE", "JWT_SECRET", "NEXUSCRM_NC_MASTER_KEY", "GH_TOKEN", "NEXUS_ADMIN_ONLINE_PASSWORD", ] missing = [key for key in required_keys if not updates[key]] if missing: raise common.DeployError(f"missing required env for multi_prod prepare: {', '.join(missing)}") validate_prod_tidb_binding(updates) remove_keys = { *RETIRED_PROD_CONTROL_PLANE_RUNTIME_KEYS, # Scrub the legacy public-DCR allowlist; the approved model uses only # administrator-provisioned Connected Apps. "NEXUSCRM_DYNAMIC_CLIENT_REDIRECT_HOSTS", "RELEASE_TUPLE_ID", "HUBSPOT_INGRESS_NGINX_BLOCK", "DEPLOY_MULTI_PROD_HUBSPOT_QUIESCE", "DEPLOY_MULTI_PROD_HUBSPOT_ADMISSION_GATE", "DEPLOY_CONNECTOR_INGRESS_GATE_TTL_SECONDS", "PROD_RESET_SOURCE_RELEASE_TAG", "PROD_RESET_SOURCE_MARKER_SHA256", "PROD_RESET_SOURCE_RUNTIME_ENV_SHA256", "PROD_RESET_SOURCE_BACKEND_PORT", "PROD_RESET_SOURCE_NGINX_CONF_PATH", "PROD_RESET_SOURCE_CAPTURED_AT", "PROD_BACKUP_RESET_RECEIPT_TTL_SECONDS", } if not smtp_updates: remove_keys.update(SMTP_RUNTIME_KEYS) if not connected_mail_oauth: remove_keys.update(CONNECTED_MAIL_OAUTH_ENV_KEYS) update_env_file(env_file, updates, remove_keys=remove_keys) write_deploy_summary( deploy_artifact_dir / "prepare.env", { "ENV_NAME": args.env, "DEPLOY_MODE": "release", "MODE": "multi-version", "RUNTIME_ENV_FILE": str(env_file), "ENV_ROOT": str(version_root), "DEPLOY_ID": deploy_id, "SKIP_BOOTSTRAP": updates["SKIP_BOOTSTRAP"], "NEXUSCRM_RUNTIME_STANDBY": updates[ "NEXUSCRM_RUNTIME_STANDBY" ], "NEXUSCRM_PUBLIC_OAUTH_PKCE_ENABLED": updates["NEXUSCRM_PUBLIC_OAUTH_PKCE_ENABLED"], "NEXUSCRM_SCIM_INPROCESS_ENABLED": updates["NEXUSCRM_SCIM_INPROCESS_ENABLED"], "RELEASE_TAG": release_config["release_tag"], "DEPLOYMENT_AUTHORITY_MODE": release_config["authority"], "RELEASE_REFS_SOURCE": release_config["release_refs_source"], "TAG_TARGET_COMMIT": release_config["tag_target_commit"], "RELEASE_EXECUTOR_COMMIT": release_config["executor_commit"], "LINEAGE_PATH": release_config["lineage_path"], "DOWNSTREAM_REF": release_config["downstream_ref"], "UPSTREAM_REF": release_config["upstream_ref"], "OFFICIAL_APP_PACKS_REF": release_config["official_app_packs_ref"], "PROD_CANDIDATE_BUNDLE_ARTIFACT_NAME": updates[ "PROD_CANDIDATE_BUNDLE_ARTIFACT_NAME" ], "PROD_CANDIDATE_BUNDLE_ARCHIVE_RELATIVE_PATH": updates[ "PROD_CANDIDATE_BUNDLE_ARCHIVE_RELATIVE_PATH" ], "PROD_CANDIDATE_BUNDLE_ARCHIVE_SHA256": updates[ "PROD_CANDIDATE_BUNDLE_ARCHIVE_SHA256" ], "PROD_CANDIDATE_BUNDLE_MANIFEST_SHA256": updates[ "PROD_CANDIDATE_BUNDLE_MANIFEST_SHA256" ], "PROD_CANDIDATE_CONNECT_MCP_URL": updates[ "PROD_CANDIDATE_CONNECT_MCP_URL" ], "PROD_EXPECTED_APP_SHELL_CONFIG_SHA256": updates["PROD_EXPECTED_APP_SHELL_CONFIG_SHA256"], "PROD_PROMOTION_AUTHORITY_SHA256": updates["PROD_PROMOTION_AUTHORITY_SHA256"], "PROD_ORDINARY_PROMOTION_CONTRACT_SHA256": updates[ "PROD_ORDINARY_PROMOTION_CONTRACT_SHA256" ], "PROD_ORDINARY_SHARED_STATE_DIGEST": updates[ "PROD_ORDINARY_SHARED_STATE_DIGEST" ], "PROD_ORDINARY_PROMOTION_ENFORCEMENT_STATUS": updates[ "PROD_ORDINARY_PROMOTION_ENFORCEMENT_STATUS" ], "PROD_RELEASE_MODE": updates["PROD_RELEASE_MODE"], "PROD_ORDINARY_ACTIVE_BASELINE_STATUS": updates[ "PROD_ORDINARY_ACTIVE_BASELINE_STATUS" ], "PROD_ORDINARY_ACTIVE_BASELINE_MATCH": updates[ "PROD_ORDINARY_ACTIVE_BASELINE_MATCH" ], "PROD_ORDINARY_ACTIVE_BASELINE_RECEIPT_SHA256": updates[ "PROD_ORDINARY_ACTIVE_BASELINE_RECEIPT_SHA256" ], "PROD_ORDINARY_MIGRATED_BASELINE_STATUS": updates[ "PROD_ORDINARY_MIGRATED_BASELINE_STATUS" ], "PROD_ORDINARY_MIGRATED_BASELINE_RECEIPT_SHA256": updates[ "PROD_ORDINARY_MIGRATED_BASELINE_RECEIPT_SHA256" ], "BACKEND_PORT": updates["BACKEND_PORT"], "BACKEND_PORT_SLOTS": updates["BACKEND_PORT_SLOTS"], "BACKEND_GO_BUILD_TAGS": updates["BACKEND_GO_BUILD_TAGS"], "NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK": updates[ "NEXUSCRM_CONNECTOR_INBOUND_WORKERS_PER_PACK" ], "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS": updates[ "NEXUSCRM_CONNECTOR_INBOUND_MAX_PROCESSING_ROWS" ], "NEXUSCRM_CONNECTOR_WORKITEM_CLAIM_ENABLED": updates[ "NEXUSCRM_CONNECTOR_WORKITEM_CLAIM_ENABLED" ], "NEXUSCRM_CONNECTOR_LIVE_QUEUE_STATS_ENABLED": updates[ "NEXUSCRM_CONNECTOR_LIVE_QUEUE_STATS_ENABLED" ], "CONNECTOR_PUSH_ASYNC_ENABLED": updates["CONNECTOR_PUSH_ASYNC_ENABLED"], "CONNECTOR_PUSH_WORKER_ENABLED": updates["CONNECTOR_PUSH_WORKER_ENABLED"], "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS": updates[ "NEXUSCRM_CONNECTOR_VERIFIED_INGRESS_RECEIPTS" ], "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_LIMIT": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_LIMIT" ], "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_LIMIT": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_LIMIT" ], "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_MS": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_VERIFICATION_WAIT_MS" ], "NEXUSCRM_CONNECTOR_WEBHOOK_RETRY_AFTER_SECONDS": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_RETRY_AFTER_SECONDS" ], "DEPLOY_CONNECTOR_INGRESS_RETRY_AFTER_SECONDS": updates[ "DEPLOY_CONNECTOR_INGRESS_RETRY_AFTER_SECONDS" ], "RATE_LIMIT_RPS": updates["RATE_LIMIT_RPS"], "RATE_LIMIT_BURST": updates["RATE_LIMIT_BURST"], "SMTP_ENABLED": str(bool(smtp_updates)).lower(), "NGINX_CONF_NAME": updates["NGINX_CONF_NAME"], "HUBSPOT_ORDINARY_WEBHOOK_MODE": updates["HUBSPOT_ORDINARY_WEBHOOK_MODE"], "HUBSPOT_WORKFLOW_EXECUTE_MODE": updates["HUBSPOT_WORKFLOW_EXECUTE_MODE"], "EXP_HUBSPOT_WORKFLOW_ACTION_ID": workflow_action_id, "EXP_HUBSPOT_WORKFLOW_ACTION_POLICIES_JSON": updates[ "EXP_HUBSPOT_WORKFLOW_ACTION_POLICIES_JSON" ], "EXP_HUBSPOT_WORKFLOW_ACTION_QUEUE_DB": updates[ "EXP_HUBSPOT_WORKFLOW_ACTION_QUEUE_DB" ], "EXP_HUBSPOT_WORKFLOW_ACTION_REPLAY_RPS": updates[ "EXP_HUBSPOT_WORKFLOW_ACTION_REPLAY_RPS" ], "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_ENDPOINTS" ], "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS": updates[ "NEXUSCRM_CONNECTOR_WEBHOOK_ACK_DROP_TIMEOUT_MS" ], "NGINX_LISTEN_PORT": updates["NGINX_LISTEN_PORT"], "NGINX_LISTEN_SUFFIX": updates["NGINX_LISTEN_SUFFIX"], "TIDB_HOST": updates["TIDB_HOST"], "TIDB_PORT": updates["TIDB_PORT"], "TIDB_USER": updates["TIDB_USER"], }, ) return env_file def write_github_output(payload: dict[str, str]) -> None: output_path = os.environ.get("GITHUB_OUTPUT", "").strip() if not output_path: return with Path(output_path).open("a", encoding="utf-8") as handle: for key, value in payload.items(): handle.write(f"{key}={value}\n") def main() -> int: args = parse_args() if args.validate_retained_runtime_env_file is not None: expected_executor_commit = str(args.expected_executor_commit or "").strip() if not expected_executor_commit: raise common.DeployError( "cannot resume: --expected-executor-commit is required with " "--validate-retained-runtime-env-file" ) workspace_repo_root = Path( os.environ.get("GITHUB_WORKSPACE", Path.cwd()) ).expanduser().resolve() release_config = validate_retained_runtime_env_contract( release_tag=args.release_tag, repo_root=workspace_repo_root, runtime_env_path=args.validate_retained_runtime_env_file.expanduser().resolve(), expected_executor_commit=expected_executor_commit, ) retained_runtime = common.load_env_file( args.validate_retained_runtime_env_file.expanduser().resolve() ) if ( str(retained_runtime.get("PUBLIC_BASE_URL", "")).strip() != args.public_base_url or release_config["candidate_connect_mcp_url"] != f"{args.public_base_url.rstrip('/')}/mcp" ): raise common.DeployError( "cannot resume: retained Prod public URL does not match candidate Connect URL" ) baseline_comparison = validate_retained_ordinary_prod_admission( runtime_env_path=args.validate_retained_runtime_env_file.expanduser().resolve(), release_config=release_config, ) refresh_retained_runtime_secret_alias( args.validate_retained_runtime_env_file.expanduser().resolve() ) print( json.dumps( { "retained_runtime_contract_validated": True, "release_tag": release_config["release_tag"], "tag_target_commit": release_config["tag_target_commit"], "ordinary_prod_baseline_status": baseline_comparison[ "status" ], "ordinary_prod_baseline_receipt_sha256": ( baseline_comparison["receipt_sha256"] ), }, indent=2, ensure_ascii=False, ) ) return 0 if args.expected_executor_commit: raise common.DeployError( "--expected-executor-commit requires " "--validate-retained-runtime-env-file" ) env_file = prepare_runtime_env(args) envv = common.load_env_file(env_file) payload = { "runtime_env_file": str(env_file), "deploy_id": str(envv.get("DEPLOY_ID", "")).strip(), "resolved_release_tag": str(envv.get("RELEASE_TAG", "")).strip(), "deployment_authority_mode": str(envv.get("DEPLOYMENT_AUTHORITY_MODE", "")).strip(), "release_refs_source": str(envv.get("RELEASE_REFS_SOURCE", "")).strip(), "tag_target_commit": str(envv.get("TAG_TARGET_COMMIT", "")).strip(), "lineage_path": str(envv.get("LINEAGE_PATH", "")).strip(), "resolved_downstream_ref": str(envv.get("DOWNSTREAM_REF", "")).strip(), "resolved_upstream_ref": str(envv.get("UPSTREAM_REF", "")).strip(), "resolved_official_app_packs_ref": str(envv.get("OFFICIAL_APP_PACKS_REF", "")).strip(), "skip_bootstrap": str(envv.get("SKIP_BOOTSTRAP", "")).strip(), "candidate_bundle_artifact_name": str( envv.get("PROD_CANDIDATE_BUNDLE_ARTIFACT_NAME", "") ).strip(), "candidate_bundle_source_run_id": str( envv.get("STAGING_RELEASE_RUN_ID", "") ).strip(), "candidate_bundle_download_root": str( envv.get("PROD_CANDIDATE_BUNDLE_DOWNLOAD_ROOT", "") ).strip(), "candidate_bundle_archive_path": str( envv.get("PROD_CANDIDATE_BUNDLE_ARCHIVE_PATH", "") ).strip(), "candidate_bundle_archive_sha256": str( envv.get("PROD_CANDIDATE_BUNDLE_ARCHIVE_SHA256", "") ).strip(), } write_github_output(payload) print(json.dumps(payload, indent=2, ensure_ascii=False)) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except common.DeployError as exc: common.log(f"failed: {exc}") raise SystemExit(1)