#!/usr/bin/env python3 from __future__ import annotations import argparse import csv import json import urllib.parse import urllib.request from collections import Counter from datetime import date from pathlib import Path from typing import Any DEFAULT_LOGIN_DOMAIN = "login.salesforce.com" def parse_args() -> argparse.Namespace: script_dir = Path(__file__).resolve().parent parser = argparse.ArgumentParser(description="Build a review report comparing Salesforce object visibility vs current S3 export coverage.") parser.add_argument("--config", required=True, help="Path to Salesforce admin config file.") parser.add_argument( "--export-config", default=str(script_dir / "configs" / "daily_full_export.json"), help="Path to export JSON config.", ) parser.add_argument( "--risk-inventory", default=str(script_dir / "configs" / "source_export_risk_inventory.json"), help="Path to special export inventory JSON.", ) parser.add_argument( "--output-dir", default=str(script_dir / "reports" / date.today().isoformat()), help="Directory where markdown and CSV outputs are written.", ) return parser.parse_args() def load_kv_config(path: Path) -> dict[str, str]: data: dict[str, str] = {} for raw in path.read_text(encoding="utf-8").splitlines(): line = raw.strip() if not line or line.startswith("#") or ":" not in line: continue key, value = line.split(":", 1) data[key.strip()] = value.strip() return data def load_json(path: Path) -> dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"{path} must be a JSON object") return payload def sf_login(config: dict[str, str]) -> tuple[str, str]: token_url = f"https://{config.get('tokenDomain') or DEFAULT_LOGIN_DOMAIN}/services/oauth2/token" form = urllib.parse.urlencode( { "grant_type": "password", "client_id": config["clientId"], "client_secret": config["clientSecret"], "username": config["username"], "password": config["password"], } ).encode("utf-8") req = urllib.request.Request(token_url, data=form, method="POST") with urllib.request.urlopen(req, timeout=30) as resp: payload = json.loads(resp.read().decode("utf-8")) return str(payload["instance_url"]), str(payload["access_token"]) def fetch_sobjects(instance_url: str, access_token: str) -> list[dict[str, Any]]: url = f"{instance_url}/services/data/v63.0/sobjects" req = urllib.request.Request(url, headers={"Authorization": f"Bearer {access_token}"}) with urllib.request.urlopen(req, timeout=120) as resp: payload = json.loads(resp.read().decode("utf-8")) objects = payload.get("sobjects", []) return [item for item in objects if isinstance(item, dict)] def recommended_priority(name: str, export_status: str, queryable: bool, retrieveable: bool, count_hint: int | None) -> str: if export_status in {"covered_bulk", "covered_special"}: return "covered" if export_status == "source_blocked": return "blocked" if not queryable or not retrieveable: return "ignore" p0 = { "CampaignMemberStatus", "EmailMessageRelation", "EventRelation", "AccountContactRelation", "AcceptedEventRelation", "ProcessInstance", "ProcessInstanceStep", "ProcessInstanceWorkitem", "ContentDistribution", } p1 = { "ContentVersionHistory", "ContentWorkspace", "Document", "Note", "EntitySubscription", "systemParameter__c", } if name in p0: return "P0" if name in p1: return "P1" if count_hint is not None and count_hint > 0: return "P2" return "review" def main() -> int: args = parse_args() output_dir = Path(args.output_dir).expanduser().resolve() output_dir.mkdir(parents=True, exist_ok=True) export_config = load_json(Path(args.export_config).expanduser().resolve()) risk_inventory = load_json(Path(args.risk_inventory).expanduser().resolve()) sf_config = load_kv_config(Path(args.config).expanduser().resolve()) instance_url, access_token = sf_login(sf_config) sobjects = fetch_sobjects(instance_url, access_token) profile = set(export_config["profiles"]["full_phase1_source_snapshot"]) configured = {str(item["name"]): item for item in export_config["objects"] if isinstance(item, dict) and item.get("name")} special_exports = { str(item["name"]): item for item in risk_inventory.get("objects", []) if isinstance(item, dict) and item.get("classification") == "special_export" and item.get("name") } source_blocked = { str(item["name"]): item for item in risk_inventory.get("objects", []) if isinstance(item, dict) and item.get("classification") == "source_unsupported" and item.get("name") } manual_count_hints = { "AcceptedEventRelation": 3934, "CampaignMemberStatus": 4662, "EventRelation": 12492, "EmailMessageRelation": 207800, "AccountContactRelation": 7042, "ProcessInstance": 2678, "ProcessInstanceStep": 10808, "ProcessInstanceWorkitem": 282, "ContentDistribution": 3500, "ContentVersionHistory": 177, "Document": 28, "ContentWorkspace": 1, "Note": 38, "EntitySubscription": 131, "systemParameter__c": 3, "Case": 1, "CaseComment": 0, "CaseContactRole": 0, "CaseHistory": 0, "Asset": 0, "QuoteDocument": 0, "FeedComment": 553, "AccountFeed": 238, "CampaignFeed": 298, "ContactFeed": 241, "EventFeed": 1604, "OpportunityFeed": 714, "OrderFeed": 39, "Product2Feed": 0, "QuoteFeed": 2718, "ContractFeed": 29, "ContentDocumentHistory": 162, "AccountHistory": 169534, "CampaignHistory": 0, "ContactHistory": 19584, "LeadFeed": 62741, "ContentDocumentFeed": 2, "ContractHistory": 469, "Product2History": 12, "OpportunityHistory": 21541, "OrderHistory": 16293, "Idea": 0, "Individual": 0, "Macro": 0, "Partner": 0, "TopicAssignment": 4, } rows: list[dict[str, Any]] = [] for item in sorted(sobjects, key=lambda x: str(x.get("name", ""))): name = str(item.get("name") or "") queryable = bool(item.get("queryable")) retrieveable = bool(item.get("retrieveable")) if name in source_blocked: export_status = "source_blocked" elif name in special_exports: export_status = "covered_special" elif name in profile and configured.get(name, {}).get("enabled", True): export_status = "covered_bulk" else: export_status = "not_covered" if (queryable and retrieveable and not item.get("deprecatedAndHidden")) else "not_accessible" rows.append( { "name": name, "label": str(item.get("label") or ""), "custom": bool(item.get("custom")), "queryable": queryable, "retrieveable": retrieveable, "replicateable": bool(item.get("replicateable")), "deprecated_and_hidden": bool(item.get("deprecatedAndHidden")), "profile_selected": name in profile, "object_configured": name in configured, "enabled_in_base_bulk": bool(configured.get(name, {}).get("enabled", False)), "export_status": export_status, "special_manifest": str(special_exports.get(name, {}).get("manifest") or ""), "count_hint": manual_count_hints.get(name, ""), "recommended_priority": recommended_priority( name, export_status, queryable, retrieveable, manual_count_hints.get(name), ), } ) csv_path = output_dir / "salesforce_object_export_coverage.csv" with csv_path.open("w", encoding="utf-8", newline="") as fh: writer = csv.DictWriter( fh, fieldnames=[ "name", "label", "custom", "queryable", "retrieveable", "replicateable", "deprecated_and_hidden", "profile_selected", "object_configured", "enabled_in_base_bulk", "export_status", "special_manifest", "count_hint", "recommended_priority", ], ) writer.writeheader() for row in rows: writer.writerow(row) export_counter = Counter(str(row["export_status"]) for row in rows) review_candidates = [ row for row in rows if row["recommended_priority"] in {"P0", "P1", "P2"} and row["export_status"] == "not_covered" ] review_candidates.sort(key=lambda row: (str(row["recommended_priority"]), -int(row["count_hint"] or 0), str(row["name"]))) md_path = output_dir / "source_export_review.md" lines = [ "# Salesforce Source Export Review", "", f"- Generated from admin describe on `{date.today().isoformat()}`", f"- Source org object count: `{len(rows)}`", f"- Current source snapshot profile: `full_phase1_source_snapshot`", f"- Coverage CSV: `{csv_path.name}`", "", "## Coverage Summary", "", f"- `covered_bulk`: `{export_counter.get('covered_bulk', 0)}`", f"- `covered_special`: `{export_counter.get('covered_special', 0)}`", f"- `source_blocked`: `{export_counter.get('source_blocked', 0)}`", f"- `not_covered`: `{export_counter.get('not_covered', 0)}`", f"- `not_accessible`: `{export_counter.get('not_accessible', 0)}`", "", "## Highest Priority Uncovered Objects", "", "| Object | Priority | Count Hint | Notes |", "| --- | --- | ---: | --- |", ] for row in review_candidates[:30]: notes = [] if row["custom"]: notes.append("custom") if not row["replicateable"]: notes.append("non-replicateable") lines.append( f"| {row['name']} | {row['recommended_priority']} | {row['count_hint'] or ''} | {', '.join(notes) or 'queryable + retrieveable'} |" ) lines.extend( [ "", "## Review Notes", "", "- `covered_special` means the object is exported through a dedicated non-Bulk path.", "- `source_blocked` means the object is not available in the current source org/API surface even for the admin account.", "- `count_hint` is only populated for objects that were manually probed during the freeze closeout work.", ] ) md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"[ok] csv={csv_path}") print(f"[ok] markdown={md_path}") return 0 if __name__ == "__main__": raise SystemExit(main())