from types import SimpleNamespace import yaml from investibook.db import queries as dbq def test_promote_run_to_repo_baseline_writes_manifest_and_syncs_db(tmp_path, monkeypatch): from investibook.core.eval import eval_baseline as baseline_module db_path = tmp_path / "investibook.db" registry_root = tmp_path / "regression" data_dir = tmp_path / "data" (registry_root / "baselines").mkdir(parents=True) monkeypatch.setattr(dbq, "DB_PATH", db_path) monkeypatch.setattr(baseline_module, "DATA_DIR", data_dir) monkeypatch.setattr(baseline_module, "load_suite", lambda *args, **kwargs: SimpleNamespace( id="core-daily-demo", suite_version="suite-v1", data={"baseline_eligible": True}, )) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-1", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-1", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-01T00:00:00+00:00", "finished_at": "2026-05-01T00:01:00+00:00", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", } }, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-1", "run_id": "evalrun-1", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-01T00:00:00+00:00", "finished_at": "2026-05-01T00:01:00+00:00", "output_text": "restart", "output": {"workspace_id": "ws-eval-1"}, "summary": { "workspace_id": "ws-eval-1", "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, }, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-1", "case_run_id": "evalcase-1", "judgment_verdict": "pass", "primary_failure_code": "", "score": 100, "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) audit_run_dir = data_dir / "eval_audits" / "evalrun-1" (audit_run_dir / "evalcase-1" / "evidences").mkdir(parents=True) (audit_run_dir / "report.txt").write_text("summary\n", encoding="utf-8") (audit_run_dir / "report_detail.txt").write_text("detail\n", encoding="utf-8") (audit_run_dir / "evalcase-1" / "semantic_context_v1.json").write_text("{}\n", encoding="utf-8") (audit_run_dir / "evalcase-1" / "semantic_packet_v1.json").write_text("{}\n", encoding="utf-8") (audit_run_dir / "evalcase-1" / "runtime").mkdir(parents=True, exist_ok=True) (audit_run_dir / "evalcase-1" / ".opencode").mkdir(parents=True, exist_ok=True) (audit_run_dir / "evalcase-1" / ".opencode" / "telemetry.log").write_text("telemetry\n", encoding="utf-8") (audit_run_dir / "evalcase-1" / "runtime_context.json").write_text( '{"outputs":{"reference":{"path":"runtime/reference.json","data":{"cluster_id":"c-1"}}}}\n', encoding="utf-8", ) (audit_run_dir / "evalcase-1" / "runtime" / "reference.json").write_text( '{"cluster_id":"c-1"}\n', encoding="utf-8", ) (audit_run_dir / "evalcase-1" / "evidences" / "workspace_evidence_report.json").write_text( "{}\n", encoding="utf-8", ) (audit_run_dir / "evalcase-1" / "evidences" / "ignored.bin").write_bytes(b"\x00\x01") manifest = baseline_module.promote_run_to_repo_baseline( "evalrun-1", created_by="tester@example.com", notes="accept current behavior", registry_root=registry_root, ) assert manifest["suite_id"] == "core-daily-demo" assert manifest["created_by"] == "tester@example.com" assert manifest["runtime"]["resolved_model"] == "openai/gpt-5.2" assert "started_at" not in manifest assert "finished_at" not in manifest assert (registry_root / "baselines" / "core-daily-demo.yaml").exists() assert manifest["runtime_export"]["dir"].endswith("baselines/runs/evalrun-1") export_dir = registry_root / "baselines" / "runs" / "evalrun-1" assert (export_dir / "report.txt").exists() assert (export_dir / "report_detail.txt").exists() assert (export_dir / "manifest.yaml").read_text(encoding="utf-8").find("resolved_model: openai/gpt-5.2") != -1 export_manifest = yaml.safe_load((export_dir / "manifest.yaml").read_text(encoding="utf-8")) assert export_manifest["started_at"] == "2026-05-01T00:00:00+00:00" assert export_manifest["finished_at"] == "2026-05-01T00:01:00+00:00" assert (export_dir / "cases" / "component-restart-during-incident-window" / "case_result.yaml").exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / "output.txt").exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / "judgment.json").exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / "semantic_context_v1.json").exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / "semantic_packet_v1.json").exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / "runtime_context.json").exists() assert ( export_dir / "cases" / "component-restart-during-incident-window" / "runtime" / "reference.json" ).exists() assert (export_dir / "cases" / "component-restart-during-incident-window" / ".opencode" / "telemetry.log").exists() assert ( export_dir / "cases" / "component-restart-during-incident-window" / "evidences" / "workspace_evidence_report.json" ).exists() assert not ( export_dir / "cases" / "component-restart-during-incident-window" / "evidences" / "ignored.bin" ).exists() active = dbq._db_get_active_eval_baseline("core-daily-demo", "suite-v1") assert active is not None assert active["promoted_from_run_id"] == "evalrun-1" def test_rejudge_exported_baseline_case_uses_exported_case_dir(tmp_path, monkeypatch): from investibook.core.eval import eval_baseline as baseline_module registry_root = tmp_path / "regression" case_dir = registry_root / "baselines" / "runs" / "evalrun-1" / "cases" / "component-restart-during-incident-window" case_dir.mkdir(parents=True) (case_dir / "output.txt").write_text("final diagnosis\n", encoding="utf-8") seen: dict[str, object] = {} monkeypatch.setattr( baseline_module, "load_case", lambda case_id, registry_root=None: SimpleNamespace(id=case_id, data={}), ) def _fake_judge_case_output(case, *, output_text, workspace_id=None, audit_dir=None): seen["case"] = case seen["output_text"] = output_text seen["workspace_id"] = workspace_id seen["audit_dir"] = audit_dir return {"judgment_verdict": "pass", "score": 100, "primary_failure_code": None} monkeypatch.setattr(baseline_module, "judge_case_output", _fake_judge_case_output) payload = baseline_module.rejudge_exported_baseline_case( run_id="evalrun-1", case_id="component-restart-during-incident-window", registry_root=registry_root, ) assert payload["case_id"] == "component-restart-during-incident-window" assert payload["judgment"]["judgment_verdict"] == "pass" assert seen["output_text"] == "final diagnosis\n" assert seen["workspace_id"] is None assert str(seen["audit_dir"]).endswith("cases/component-restart-during-incident-window") def test_promote_run_to_repo_baseline_falls_back_to_case_runtime_metadata(tmp_path, monkeypatch): from investibook.core.eval import eval_baseline as baseline_module db_path = tmp_path / "investibook.db" registry_root = tmp_path / "regression" data_dir = tmp_path / "data" (registry_root / "baselines").mkdir(parents=True) monkeypatch.setattr(dbq, "DB_PATH", db_path) monkeypatch.setattr(baseline_module, "DATA_DIR", data_dir) monkeypatch.setattr( baseline_module, "load_suite", lambda *args, **kwargs: SimpleNamespace( id="core-daily-demo", suite_version="suite-v1", data={"baseline_eligible": True}, ), ) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-2", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-2", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-01T00:00:00+00:00", "finished_at": "2026-05-01T00:01:00+00:00", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": {}, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-2", "run_id": "evalrun-2", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-01T00:00:00+00:00", "finished_at": "2026-05-01T00:01:00+00:00", "output_text": "restart", "output": {"workspace_id": "ws-eval-2"}, "summary": { "workspace_id": "ws-eval-2", "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, }, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-2", "case_run_id": "evalcase-2", "judgment_verdict": "pass", "primary_failure_code": "", "score": 100, "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) audit_run_dir = data_dir / "eval_audits" / "evalrun-2" (audit_run_dir / "evalcase-2" / "evidences").mkdir(parents=True) (audit_run_dir / "report.txt").write_text("summary\n", encoding="utf-8") (audit_run_dir / "report_detail.txt").write_text("detail\n", encoding="utf-8") (audit_run_dir / "evalcase-2" / "semantic_context_v1.json").write_text("{}\n", encoding="utf-8") (audit_run_dir / "evalcase-2" / "semantic_packet_v1.json").write_text("{}\n", encoding="utf-8") (audit_run_dir / "evalcase-2" / ".opencode").mkdir(parents=True, exist_ok=True) (audit_run_dir / "evalcase-2" / ".opencode" / "telemetry.log").write_text("telemetry\n", encoding="utf-8") (audit_run_dir / "evalcase-2" / "evidences" / "workspace_evidence_report.json").write_text( "{}\n", encoding="utf-8", ) manifest = baseline_module.promote_run_to_repo_baseline( "evalrun-2", created_by="tester@example.com", notes="runtime fallback", registry_root=registry_root, ) assert manifest["runtime"]["agent_type"] == "opencode" assert manifest["runtime"]["resolved_model"] == "openai/gpt-5.2" def test_promote_run_to_repo_baseline_derives_run_times_from_case_runs(tmp_path, monkeypatch): from investibook.core.eval import eval_baseline as baseline_module db_path = tmp_path / "investibook.db" registry_root = tmp_path / "regression" data_dir = tmp_path / "data" (registry_root / "baselines").mkdir(parents=True) monkeypatch.setattr(dbq, "DB_PATH", db_path) monkeypatch.setattr(baseline_module, "DATA_DIR", data_dir) monkeypatch.setattr( baseline_module, "load_suite", lambda *args, **kwargs: SimpleNamespace( id="core-daily-demo", suite_version="suite-v1", data={"baseline_eligible": True}, ), ) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-3", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-3", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "", "finished_at": "", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": {}, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-3", "run_id": "evalrun-3", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-01T00:02:00+00:00", "finished_at": "2026-05-01T00:04:00+00:00", "output_text": "restart", "output": {"workspace_id": "ws-eval-3"}, "summary": {"workspace_id": "ws-eval-3"}, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-3", "case_run_id": "evalcase-3", "judgment_verdict": "pass", "primary_failure_code": "", "score": 100, "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) audit_run_dir = data_dir / "eval_audits" / "evalrun-3" (audit_run_dir / "evalcase-3" / "evidences").mkdir(parents=True) (audit_run_dir / "report.txt").write_text("summary\n", encoding="utf-8") (audit_run_dir / "report_detail.txt").write_text("detail\n", encoding="utf-8") (audit_run_dir / "evalcase-3" / "evidences" / "workspace_evidence_report.json").write_text( "{}\n", encoding="utf-8", ) manifest = baseline_module.promote_run_to_repo_baseline( "evalrun-3", created_by="tester@example.com", notes="derive run time bounds", registry_root=registry_root, ) assert "started_at" not in manifest assert "finished_at" not in manifest export_manifest = yaml.safe_load( (registry_root / "baselines" / "runs" / "evalrun-3" / "manifest.yaml").read_text(encoding="utf-8") ) assert export_manifest["started_at"] == "2026-05-01T00:02:00+00:00" assert export_manifest["finished_at"] == "2026-05-01T00:04:00+00:00" def test_promote_run_to_repo_baseline_preserves_other_runtime_variants(tmp_path, monkeypatch): from investibook.core.eval import eval_baseline as baseline_module db_path = tmp_path / "investibook.db" registry_root = tmp_path / "regression" data_dir = tmp_path / "data" (registry_root / "baselines").mkdir(parents=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-claude suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-claude created_at: 2026-05-10T00:00:00+00:00 created_by: tester@example.com notes: existing claude baseline runtime: agent_type: opencode resolved_model: chat/claude-standard summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-claude workspace_id: ws-claude execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 95 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) monkeypatch.setattr(dbq, "DB_PATH", db_path) monkeypatch.setattr(baseline_module, "DATA_DIR", data_dir) monkeypatch.setattr( baseline_module, "load_suite", lambda *args, **kwargs: SimpleNamespace( id="core-daily-demo", suite_version="suite-v1", data={"baseline_eligible": True}, ), ) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-gpt", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-4", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-11T00:00:00+00:00", "finished_at": "2026-05-11T00:01:00+00:00", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", } }, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-gpt", "run_id": "evalrun-gpt", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-11T00:00:00+00:00", "finished_at": "2026-05-11T00:01:00+00:00", "output_text": "restart", "output": {"workspace_id": "ws-eval-gpt"}, "summary": { "workspace_id": "ws-eval-gpt", "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, }, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-gpt", "case_run_id": "evalcase-gpt", "judgment_verdict": "pass", "primary_failure_code": "", "score": 100, "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) audit_run_dir = data_dir / "eval_audits" / "evalrun-gpt" (audit_run_dir / "evalcase-gpt" / "evidences").mkdir(parents=True) (audit_run_dir / "report.txt").write_text("summary\n", encoding="utf-8") (audit_run_dir / "report_detail.txt").write_text("detail\n", encoding="utf-8") (audit_run_dir / "evalcase-gpt" / "evidences" / "workspace_evidence_report.json").write_text( "{}\n", encoding="utf-8", ) manifest = baseline_module.promote_run_to_repo_baseline( "evalrun-gpt", created_by="tester@example.com", notes="add gpt baseline", registry_root=registry_root, ) assert manifest["baseline_id"] == "baseline-evalrun-gpt" assert manifest["runtime"]["resolved_model"] == "openai/gpt-5.2" variants = manifest["baseline_variants"] assert len(variants) == 2 assert variants[0]["runtime"]["resolved_model"] == "openai/gpt-5.2" assert variants[1]["runtime"]["resolved_model"] == "chat/claude-standard"