"""Print the epic #4863 success-criteria measurables from an audit output directory. Reads /.json, /_all_fields.csv and /twins.json. Maps to the epic's success criteria: SC-1 layout cleanliness -> dead-on-layout counts (A/B buckets on layout) SC-2 agent surface -> off-layout dead+writable counts, collision groups SC-3 semantics coverage -> HelpText/Description fill counts SC-4 convergence -> twin divergence counters, collision groups Usage: metrics_summary.py [OBJECT ...] [--out DIR] """ from __future__ import annotations import collections import csv import json import re import sys from common import parse_args def main() -> None: objects, out_dir, _ = parse_args(sys.argv[1:]) # The manifest records what the probe in THIS output directory actually # covered. Files alone are not proof of freshness: a stale CSV left by an # earlier run must not turn a partial run green. manifest_file = out_dir / "manifest.json" if not manifest_file.exists(): print(f"ERROR: {manifest_file} not found — run probe_full_audit.py first") raise SystemExit(2) manifest = json.loads(manifest_file.read_text()) probed = set(manifest.get("probed", [])) probe_skipped = set(manifest.get("skipped", [])) print("# Field-governance metrics (epic #4863)\n") header = ( "| Object | Fields | On layout | Dead on layout (SC-1) | Off dead+writable (SC-2) | " "Semantics filled (SC-3) |" ) print(header) print("|---|---|---|---|---|---|") collision_total = 0 missing = [] skipped_absent: list[str] = [] sem_unmeasurable: list[str] = [] unknown_cohort: dict[str, int] = {} audited = 0 for obj in objects: pj = out_dir / f"{obj}.json" pc = out_dir / f"{obj}_all_fields.csv" if obj in probe_skipped: skipped_absent.append(obj) continue if obj not in probed or not (pj.exists() and pc.exists()): missing.append(obj) continue data = json.loads(pj.read_text()) uc = data.get("rows", {}).get("unknown_cohort", 0) if uc: unknown_cohort[obj] = uc rows = list(csv.DictReader(open(pc))) on = [r for r in rows if r["on_layout"] == "True"] off = [r for r in rows if r["on_layout"] != "True"] dead_on = sum(1 for r in on if r["bucket"] in ("A-empty-both", "B2-bool-never-true", "B-legacy-only")) off_dead_writable = sum( 1 for r in off if r["bucket"] in ("A-empty-both", "B2-bool-never-true", "B-legacy-only") and (r["createable"] == "True" or r["updateable"] == "True") ) # Semantic columns are discovered by the probe (help_cols), not assumed here. hf = data.get("help_filled", {}) help_cols = data.get("help_cols", []) if not help_cols: sem = "UNMEASURABLE" sem_unmeasurable.append(obj) else: sem = ", ".join(f"{c}={hf[c]}" for c in help_cols) print(f"| {obj} | {len(rows)} | {len(on)} | {dead_on} | {off_dead_writable} | {sem} |") audited += 1 # collision groups (same normalized label, or API names equal modulo underscores) bylabel = collections.defaultdict(list) byname = collections.defaultdict(list) for r in rows: ln = re.sub(r"[^a-z0-9]", "", r["label"].lower()) if ln: bylabel[ln].append(r["field"]) byname[re.sub(r"(__c$)", "", r["field"].lower()).replace("_", "")].append(r["field"]) keys = {tuple(sorted(g)) for g in list(bylabel.values()) + list(byname.values()) if len(g) > 1} collision_total += len(keys) print(f"\nCollision groups across audited objects (SC-2/SC-4): {collision_total}") twins_file = out_dir / "twins.json" if twins_file.exists(): print("\nTwin divergence counters (SC-4):") for k, v in json.loads(twins_file.read_text()).items(): print(f"- {k}: {json.dumps(v)}") # Fail loudly on partial or empty coverage: the epic's acceptance gate is # "re-run the metrics, all green", and a typo'd object name or an empty # database must not read as green. failed = False if missing: print(f"\nERROR: no audit output for: {', '.join(missing)} (metrics above are partial)") failed = True if skipped_absent: print( f"\nERROR: probe skipped (absent from _System_Object): {', '.join(skipped_absent)} — " "audit coverage is partial; drop these objects or fix the database target" ) failed = True if audited == 0: print("\nERROR: no objects audited") failed = True if sem_unmeasurable: print( f"\nERROR: SC-3 unmeasurable (no HelpText/Description columns found) for: " f"{', '.join(sem_unmeasurable)}" ) failed = True if unknown_cohort: print( "\nERROR: rows with Id shapes outside 15/18/36 are excluded from all fill " f"measurement, so bucket verdicts are unreliable: {json.dumps(unknown_cohort)}" ) failed = True if failed: raise SystemExit(2) if __name__ == "__main__": main()