#!/usr/bin/env python3 """Evaluate governance memory and historical-action migration from weekly reviews.""" from __future__ import annotations import argparse import json import sys from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from backend.sql_catalog import server # noqa: E402 DEFAULT_DATASET = ROOT / "data" / "evaluation" / "manual_weekly_review_cases.json" def evaluate(dataset_path: Path) -> dict[str, Any]: dataset = json.loads(dataset_path.read_text(encoding="utf-8")) results: list[dict[str, Any]] = [] with server.get_conn() as conn: for case in dataset.get("cases", []): row = conn.execute( """ SELECT * FROM catalog_items WHERE cluster_id = ? AND sql_digest = ? ORDER BY created_at DESC LIMIT 1 """, (case["cluster_id"], case["sql_digest"]), ).fetchone() decision_action = str(case.get("decision_action") or "") if not row: results.append( { "id": case["id"], "status": "missing", "decision_action": decision_action, } ) continue item = server.row_to_catalog_item(row) posture = server.catalog_current_posture(conn, item) actual = bool(posture["review_priority"]["recommended"]) if decision_action == "create_treatment": treatment = conn.execute( """ SELECT t.id, t.state, t.stage FROM treatments t JOIN relations r ON r.from_type = 'treatment' AND r.from_id = t.id AND r.relation_type = 'treatment_for' AND r.to_type = 'catalog_item' AND r.to_id = ? ORDER BY t.created_at DESC LIMIT 1 """, (item["id"],), ).fetchone() results.append( { "id": case["id"], "status": "historical_treatment_present" if treatment else "historical_treatment_missing", "decision_action": decision_action, "current_recommended": actual, "current_priority": posture["review_priority"]["tier"], "treatment_id": treatment["id"] if treatment else None, } ) continue memory = posture.get("governance_memory") or {} decision_record = ( server.get_record(conn, str(memory.get("record_id"))) if memory.get("record_id") else None ) explains_later_change = bool( actual and memory.get("status") == "change_detected" and decision_record and decision_record["structured_fields"].get("manual_review_case_id") == case["id"] ) status = ( "governance_memory_active" if not actual and memory.get("status") == "active" else "reopened_after_review" if explains_later_change else "governance_memory_missing" ) results.append( { "id": case["id"], "status": status, "decision_action": decision_action, "current_recommended": actual, "priority": posture["review_priority"]["tier"], "posture": posture["state"], "governance_memory": memory.get("status"), "change_reasons": memory.get("trigger_reasons") or [], } ) memory_cases = [ result for result in results if result.get("decision_action") == "no_action" ] accepted_memory = [ result for result in memory_cases if result["status"] in {"governance_memory_active", "reopened_after_review"} ] treatment_cases = [ result for result in results if result.get("decision_action") == "create_treatment" ] return { "dataset": str(dataset_path), "database": str(server.DB_PATH), "summary": { "total_cases": len(results), "present": sum(result["status"] != "missing" for result in results), "missing": sum(result["status"] == "missing" for result in results), "governance_memory_active": sum(result["status"] == "governance_memory_active" for result in results), "reopened_after_review": sum(result["status"] == "reopened_after_review" for result in results), "governance_memory_missing": sum(result["status"] == "governance_memory_missing" for result in results), "historical_treatment_present": sum(result["status"] == "historical_treatment_present" for result in results), "historical_treatment_missing": sum(result["status"] == "historical_treatment_missing" for result in results), "memory_explained_rate": round(len(accepted_memory) / len(memory_cases), 4) if memory_cases else None, "treatment_migration_rate": round( sum(result["status"] == "historical_treatment_present" for result in treatment_cases) / len(treatment_cases), 4 ) if treatment_cases else None, }, "results": results, } def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=Path, default=DEFAULT_DATASET) parser.add_argument( "--strict", action="store_true", help="Exit non-zero when a present no-action case lacks memory or a historical action lacks Treatment migration.", ) args = parser.parse_args() report = evaluate(args.dataset.resolve()) print(json.dumps(report, ensure_ascii=False, indent=2)) unresolved = ( report["summary"]["governance_memory_missing"] + report["summary"]["historical_treatment_missing"] ) return 1 if args.strict and unresolved else 0 if __name__ == "__main__": raise SystemExit(main())