import json import shutil from copy import deepcopy from dataclasses import replace from pathlib import Path from types import SimpleNamespace import pytest from investibook.db import queries as dbq from investibook.core.eval.eval_baseline import load_repo_baseline from investibook.core.eval.eval_runner import environment_fingerprint from investibook.core.eval.eval_runner import create_eval_workspace from investibook.core.eval.eval_runner import _case_prompt from investibook.core.eval.eval_runner import _case_failure_detail_lines from investibook.core.eval.eval_runner import _initialize_runtime_context from investibook.core.eval.eval_runner import _render_runtime_template from investibook.core.eval.eval_runner import _run_runtime_hooks from investibook.core.eval.eval_runner import ( EvalRunnerOptions, _compute_baseline_comparison, _build_capacity_timeout_salvage, _build_expensive_query_timeout_salvage, _ensure_workspace_files, _resolve_active_baseline, _classify_acp_result, _format_eval_run_detail_report, _format_eval_run_report, _mirror_eval_audit_artifacts, _run_acp_case, _write_eval_live_dotenv, run_eval_suite, ) from investibook.core.eval.eval_registry import load_case, load_suite from .test_eval_judge import PASSING_OUTPUT def test_environment_fingerprint_is_stable_shape(): fingerprint = environment_fingerprint() assert fingerprint.startswith("local:") assert "python" in fingerprint assert "platform" in fingerprint def test_write_eval_live_dotenv_only_resets_runtime_dirs(tmp_path): workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() from investibook.core.eval import eval_runner as eval_runner_module eval_runner_module.EVAL_AUDIT_DIR = tmp_path / "audit" (workspace_dir / ".env").write_text("CUSTOM_FLAG=1\n") (workspace_dir / "evidences").mkdir() (workspace_dir / "evidences" / "stale.txt").write_text("stale\n") (workspace_dir / "clinic-downloads").mkdir() (workspace_dir / "clinic-downloads" / "stale.txt").write_text("stale\n") _write_eval_live_dotenv(workspace_dir) text = (workspace_dir / ".env").read_text() assert "CUSTOM_FLAG=1" in text assert (workspace_dir / "evidences").exists() assert not (workspace_dir / "evidences" / "stale.txt").exists() assert (workspace_dir / "clinic-downloads").exists() assert not (workspace_dir / "clinic-downloads" / "stale.txt").exists() assert (eval_runner_module.EVAL_AUDIT_DIR / "workspace").exists() def test_ensure_workspace_files_for_eval_omits_download_dir_env(monkeypatch, tmp_path): workspace_id = "ws-eval-123" workspace_dir = tmp_path / workspace_id monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path) monkeypatch.setenv("CLINIC_API_KEY", "clinic-token") def _write_stub(dest, *_args, **_kwargs): (dest / ".opencode").mkdir(parents=True, exist_ok=True) (dest / ".opencode" / "opencode.json").write_text("{}", encoding="utf-8") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", _write_stub) _ensure_workspace_files(workspace_id) text = (workspace_dir / ".env").read_text() assert "INVESTIBOOK_WORKSPACE_DIR=" in text assert 'CLINIC_DIAGNOSTIC_DOWNLOAD_DIR="' in text assert 'CLINIC_API_KEY="clinic-token"' in text def test_classify_acp_execution_timeout_is_infra_invalid(): status, invalid_subtype, failure_reason = _classify_acp_result( {"finish_reason": "timeout", "text": "", "error": "Total timeout after 1s"} ) assert status == "invalid" assert invalid_subtype == "infra" assert failure_reason == "agent_execution_timeout" def test_bounded_timeout_salvage_uses_canonical_classification_contract(): required_tag_fields = { "id", "source", "component", "metric_or_signal", "direction", "baseline", "severity", "confidence", "evidence_ref", } required_classification_fields = { "primary_class", "candidates", "rejected_classes", "classification_confidence", "missing_evidence_for_higher_confidence", } capacity = _build_capacity_timeout_salvage( evidence_agent={ "critical_evidence_hints": [ {"id": "capacity_comparison_context_present", "status": "matched", "summary": "matched"} ] } )["diagnosis"] expensive_query = _build_expensive_query_timeout_salvage( evidence_agent={ "critical_evidence_hints": [ {"id": "plan_bottleneck_signal_present", "status": "matched", "summary": "matched"} ] } )["diagnosis"] for diagnosis in (capacity, expensive_query): assert diagnosis["signal_tags"] for tag in diagnosis["signal_tags"]: assert required_tag_fields <= set(tag) assert "comparison" not in tag assert "evidence_path" not in tag assert required_classification_fields <= set(diagnosis["fault_classification"]) assert diagnosis["selected_diagnosis_skills"] assert capacity["fault_classification"]["primary_class"] == "resource_control_or_capacity" assert expensive_query["fault_classification"]["primary_class"] == "sql_plan_or_query" def test_classify_acp_error_finish_reason_is_infra_invalid(): status, invalid_subtype, failure_reason = _classify_acp_result( {"finish_reason": "error", "text": "", "error": "agent crashed"} ) assert status == "invalid" assert invalid_subtype == "infra" assert failure_reason == "acp_error" def test_classify_acp_bootstrap_timeout_is_infra_invalid(): status, invalid_subtype, failure_reason = _classify_acp_result( { "finish_reason": "error", "text": "", "error": "ACP bootstrap timeout during conn.initialize after 60s", } ) assert status == "invalid" assert invalid_subtype == "infra" assert failure_reason == "acp_bootstrap_timeout" def test_case_prompt_is_minimal_user_intent(): case = load_case("component-restart-during-incident-window") prompt = _case_prompt(case) assert prompt.startswith(case.prompt_text) assert "collect_evidence.py" in prompt assert "300000 ms (300s)" in prompt assert "组件重启" in prompt assert case.id not in prompt def test_case_prompt_renders_runtime_template_values(): case = load_case("component-restart-during-incident-window") data = deepcopy(case.data) data["prompt"]["text"] = ( "请分析集群 {{environment.cluster_id}} 在 " "{{runtime.resolved_window.start}} 到 {{runtime.resolved_window.end}} 的情况。" ) case = replace(case, data=data) runtime_context = _initialize_runtime_context(case=case, workspace_id="ws-runtime-1") runtime_context["outputs"]["resolved_window"] = { "data": { "start": "2026-04-01T10:36:00Z", "end": "2026-04-01T11:36:00Z", } } prompt = _case_prompt(case, runtime_context=runtime_context) assert "10766841556523235152" in prompt assert "2026-04-01T10:36:00Z" in prompt assert "2026-04-01T11:36:00Z" in prompt assert "{{runtime.resolved_window.start}}" not in prompt def test_case_prompt_adds_evidence_only_stage_boundary(): case = load_case("component-restart-during-incident-window") case = replace(case, data={**case.data, "evaluation": {"mode": "evidence_only"}}) prompt = _case_prompt(case) assert "Evaluation mode hint: this case is evidence_only" in prompt assert "write `evidences/agent-runs/evidence-agent.json`" in prompt assert "Do not invoke `diagnosis-analyst`" in prompt assert "do not produce a final root-cause diagnosis" in prompt def test_case_prompt_adds_diagnosis_only_stage_boundary(): case = load_case("component-restart-during-incident-window") case = replace(case, data={**case.data, "evaluation": {"mode": "diagnosis_only"}}) prompt = _case_prompt(case) assert "Evaluation mode hint: this case is diagnosis_only" in prompt assert "Use the evidence files already present in the workspace" in prompt assert "write `evidences/agent-runs/diagnosis-analyst.json`" in prompt assert "Do not run `collect_evidence.py`" in prompt assert "do not invoke `evidence-agent`" in prompt def test_render_runtime_template_leaves_unknown_placeholders_untouched(): rendered = _render_runtime_template( "cluster={{environment.cluster_id}} missing={{runtime.unknown.value}}", runtime_context={ "template_context": {"environment": {"cluster_id": "cluster-1"}}, "outputs": {}, }, ) assert rendered == "cluster=cluster-1 missing={{runtime.unknown.value}}" def test_run_runtime_hooks_executes_python_script_and_persists_outputs(tmp_path): case = load_case("component-restart-during-incident-window") case_dir = tmp_path / "case" case_dir.mkdir() script_path = case_dir / "runtime_hook.py" script_path.write_text( "\n".join( [ "import json", "import os", "import sys", "", "print(json.dumps({", ' "args": sys.argv[1:],', ' "workspace_id": os.environ["INVESTIBOOK_EVAL_WORKSPACE_ID"],', ' "workspace_dir": os.environ["INVESTIBOOK_EVAL_WORKSPACE_DIR"],', ' "runtime_context_exists": os.path.exists(' 'os.environ["INVESTIBOOK_EVAL_RUNTIME_CONTEXT_PATH"]),', "}))", ] ) + "\n", encoding="utf-8", ) data = deepcopy(case.data) data["runtime"] = { "hooks": [ { "id": "resolve-window", "stage": "before_acp", "kind": "python_script", "path": "runtime_hook.py", "output_id": "resolved_window", "args": [ "{{environment.cluster_id}}", "{{workspace.id}}", "{{runtime.seed.value}}", ], } ] } case = replace(case, path=case_dir / "case.yaml", data=data) workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() audit_dir = tmp_path / "audit" runtime_context = _initialize_runtime_context(case=case, workspace_id="ws-runtime-hooks") runtime_context["outputs"]["seed"] = {"data": {"value": "alpha"}} result = _run_runtime_hooks( case=case, stage="before_acp", workspace_id="ws-runtime-hooks", workspace_dir=workspace_dir, audit_dir=audit_dir, runtime_context=runtime_context, ) assert result == {"ok": True, "failure": None} assert runtime_context["outputs"]["resolved_window"]["data"]["args"] == [ "10766841556523235152", "ws-runtime-hooks", "alpha", ] assert runtime_context["outputs"]["resolved_window"]["data"]["workspace_id"] == "ws-runtime-hooks" assert runtime_context["outputs"]["resolved_window"]["data"]["workspace_dir"] == str(workspace_dir) assert runtime_context["outputs"]["resolved_window"]["data"]["runtime_context_exists"] is True persisted_context = json.loads((audit_dir / "runtime_context.json").read_text(encoding="utf-8")) assert persisted_context["outputs"]["resolved_window"]["path"] == "runtime/resolved_window.json" persisted_output = json.loads((audit_dir / "runtime" / "resolved_window.json").read_text(encoding="utf-8")) assert persisted_output["args"] == ["10766841556523235152", "ws-runtime-hooks", "alpha"] def test_create_eval_workspace_has_no_bound_jira_sources(tmp_path, monkeypatch): case = load_case("component-restart-during-incident-window") monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path) workspace_id = create_eval_workspace( case, user_email="eval@localhost", ) workspace_dir = tmp_path / workspace_id assert not (workspace_dir / "issues").exists() def test_resolve_active_baseline_prefers_repo_manifest(tmp_path, monkeypatch): registry_root = tmp_path / "regression" (registry_root / "baselines").mkdir(parents=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-evalrun-1 suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-1 created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-1 workspace_id: ws-eval-1 execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) monkeypatch.setattr( "investibook.core.eval.eval_runner._db_get_active_eval_baseline", lambda *_args, **_kwargs: { "id": "db-baseline", "suite_id": "core-daily-demo", "suite_version": "suite-v1", }, ) baseline = _resolve_active_baseline( "core-daily-demo", "suite-v1", registry_root=registry_root, ) assert baseline is not None assert baseline["baseline_id"] == "baseline-evalrun-1" assert baseline["source"] == "repo" def test_resolve_active_baseline_selects_matching_repo_variant_by_runtime(tmp_path, monkeypatch): registry_root = tmp_path / "regression" (registry_root / "baselines").mkdir(parents=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-claude suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-claude created_at: 2026-05-12T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: chat/claude-standard summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-claude workspace_id: ws-claude execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 90 passed_checks: 5 total_checks: 5 baseline_variants: - baseline_id: baseline-claude suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-claude created_at: 2026-05-12T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: chat/claude-standard summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-claude workspace_id: ws-claude execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 90 passed_checks: 5 total_checks: 5 - baseline_id: baseline-gpt suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-gpt created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: openai/gpt-5.2 summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-gpt workspace_id: ws-gpt execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) monkeypatch.setattr( "investibook.core.eval.eval_runner._db_get_active_eval_baseline", lambda *_args, **_kwargs: None, ) baseline = _resolve_active_baseline( "core-daily-demo", "suite-v1", registry_root=registry_root, runtime_metadata={ "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, ) assert baseline is not None assert baseline["baseline_id"] == "baseline-gpt" assert baseline["source"] == "repo" assert baseline["runtime"]["resolved_model"] == "openai/gpt-5.2" def test_load_repo_baseline_ignores_version_mismatch(tmp_path): registry_root = tmp_path / "regression" (registry_root / "baselines").mkdir(parents=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-evalrun-1 suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-1 created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-1 workspace_id: ws-eval-1 execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) baseline = load_repo_baseline( "core-daily-demo", suite_version="suite-v2", registry_root=registry_root, ) assert baseline is not None assert baseline["baseline_id"] == "baseline-evalrun-1" assert baseline["suite_version"] == "suite-v1" def test_load_repo_baseline_prefers_runtime_variant_even_when_suite_version_mismatches(tmp_path): registry_root = tmp_path / "regression" (registry_root / "baselines").mkdir(parents=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-claude suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-claude created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: chat/claude-standard summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-claude workspace_id: ws-claude execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 90 passed_checks: 5 total_checks: 5 baseline_variants: - baseline_id: baseline-gpt suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-gpt created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: openai/gpt-5.2 summary: execution_status: completed cases: - case_id: component-restart-during-incident-window case_version: case-v1 case_run_id: evalcase-gpt workspace_id: ws-gpt execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) baseline = load_repo_baseline( "core-daily-demo", suite_version="suite-v2", registry_root=registry_root, runtime_metadata={ "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, ) assert baseline is not None assert baseline["baseline_id"] == "baseline-gpt" assert baseline["runtime"]["resolved_model"] == "openai/gpt-5.2" def test_mirror_eval_audit_artifacts_copies_evidences_telemetry_and_dispatches( monkeypatch, tmp_path, ): from investibook.core.eval import eval_runner as eval_runner_module eval_runner_module.EVAL_AUDIT_DIR = tmp_path / "audit" monkeypatch.setattr( eval_runner_module, "_db_get_workspace_messages_for_eval", lambda workspace_id, source: [ { "id": "assistant-1", "role": "assistant", "parts": [ { "type": "tool", "id": "tool-child-bash", "state": { "meta": { "opencode/subagent": { "parentTaskCallId": "task-diagnosis-1", "subagentSessionId": "session-diagnosis-1", "subagentName": "diagnosis-analyst", } } }, }, { "type": "reasoning", "meta": { "opencode/subagent": { "parentTaskCallId": "task-diagnosis-1", "subagentSessionId": "session-diagnosis-1", "subagentName": "diagnosis-analyst", } }, }, { "type": "tool", "state": { "meta": { "opencode/subagent": { "parentTaskCallId": "task-incomplete", "subagentName": "investigator", } } }, }, ], } ], ) workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() (workspace_dir / ".opencode").mkdir() (workspace_dir / ".opencode" / "telemetry.log").write_text("telemetry\n") (workspace_dir / "evidences").mkdir() (workspace_dir / "evidences" / "workspace_evidence_report.json").write_text("{}\n") (workspace_dir / "clinic-downloads" / "2026-04-01" / "diagnosis-001" / "evidence").mkdir(parents=True) (workspace_dir / "clinic-downloads" / "2026-04-01" / "diagnosis-001" / "diagnosis_report.json").write_text("{}\n") ( workspace_dir / "clinic-downloads" / "2026-04-01" / "diagnosis-001" / "evidence" / "queries.json" ).write_text("{}\n") audit_dir = _mirror_eval_audit_artifacts( run_id="evalrun-1", case_run_id="evalcase-1", workspace_dir=workspace_dir, ) assert (audit_dir / ".opencode" / "telemetry.log").exists() assert (audit_dir / "evidences" / "workspace_evidence_report.json").exists() assert (audit_dir / "clinic-downloads" / "2026-04-01" / "diagnosis-001" / "diagnosis_report.json").exists() assert (audit_dir / "clinic-downloads" / "2026-04-01" / "diagnosis-001" / "evidence" / "queries.json").exists() dispatch_trace = json.loads( (audit_dir / "runtime" / "agent_dispatches.json").read_text(encoding="utf-8") ) assert dispatch_trace == { "schema_version": 1, "workspace_id": "workspace", "source": "messages_v2_acp_metadata", "subagent_executions": [ { "subagent_name": "diagnosis-analyst", "parent_task_call_id": "task-diagnosis-1", "subagent_session_id": "session-diagnosis-1", } ], } @pytest.mark.asyncio async def test_run_acp_case_releases_session_after_eval(monkeypatch): case = load_case("component-restart-during-incident-window") calls: list[str] = [] async def _fake_run_chat_and_wait(**kwargs): return ("session-1", SimpleNamespace(text="final diagnosis", finish_reason="completed", error=None)) async def _fake_release_acp_session(workspace_id: str) -> bool: calls.append(workspace_id) return True monkeypatch.setattr("investibook.api.chat.run_chat_and_wait", _fake_run_chat_and_wait) monkeypatch.setattr("investibook.core.agent_runtime.release_agent_session", _fake_release_acp_session) result = await _run_acp_case( prompt=_case_prompt(case), workspace_id="ws-eval-test", user_email="eval@localhost", timeout_s=30, idle_timeout_s=10, ) assert result["finish_reason"] == "completed" assert "ws-eval-test" in calls @pytest.mark.asyncio async def test_run_acp_case_passes_live_like_chat_args(monkeypatch): case = load_case("component-restart-during-incident-window") seen: dict[str, object] = {} async def _fake_run_chat_and_wait(**kwargs): seen.update(kwargs) return ("session-1", SimpleNamespace(text="final diagnosis", finish_reason="completed", error=None)) async def _fake_release_acp_session(workspace_id: str) -> bool: return True monkeypatch.setattr("investibook.api.chat.run_chat_and_wait", _fake_run_chat_and_wait) monkeypatch.setattr("investibook.core.agent_runtime.release_agent_session", _fake_release_acp_session) result = await _run_acp_case( prompt=_case_prompt(case), workspace_id="ws-eval-test", user_email="eval@localhost", timeout_s=15, idle_timeout_s=10, ) assert result["finish_reason"] == "completed" assert seen["timeout_s"] == 15 assert seen["idle_timeout_s"] == 10 assert seen["metrics_channel"] == "eval" assert seen["diagnosis_mode"] == "autopilot" assert seen["trigger_type"] == "evaluation" assert seen["supports_followup"] is False @pytest.mark.asyncio async def test_run_acp_case_accepts_case_specific_idle_timeout(monkeypatch): case = load_case("sustained-cpu-pressure-during-incident-window") seen: dict[str, object] = {} async def _fake_run_chat_and_wait(**kwargs): seen.update(kwargs) return ("session-1", SimpleNamespace(text="final diagnosis", finish_reason="completed", error=None)) async def _fake_release_acp_session(workspace_id: str) -> bool: return True monkeypatch.setattr("investibook.api.chat.run_chat_and_wait", _fake_run_chat_and_wait) monkeypatch.setattr("investibook.core.agent_runtime.release_agent_session", _fake_release_acp_session) result = await _run_acp_case( prompt=_case_prompt(case), workspace_id="ws-eval-test", user_email="eval@localhost", timeout_s=900, idle_timeout_s=max(300, case.acp_idle_timeout_s or 0), ) assert result["finish_reason"] == "completed" assert seen["idle_timeout_s"] == 900 @pytest.mark.asyncio async def test_run_acp_case_accepts_case_specific_total_timeout(monkeypatch): case = load_case("sustained-cpu-pressure-during-incident-window") seen: dict[str, object] = {} async def _fake_run_chat_and_wait(**kwargs): seen.update(kwargs) return ("session-1", SimpleNamespace(text="final diagnosis", finish_reason="completed", error=None)) async def _fake_release_acp_session(workspace_id: str) -> bool: return True monkeypatch.setattr("investibook.api.chat.run_chat_and_wait", _fake_run_chat_and_wait) monkeypatch.setattr("investibook.core.agent_runtime.release_agent_session", _fake_release_acp_session) result = await _run_acp_case( prompt=_case_prompt(case), workspace_id="ws-eval-test", user_email="eval@localhost", timeout_s=max(900, case.acp_timeout_s or 0), idle_timeout_s=max(300, case.acp_idle_timeout_s or 0), ) assert result["finish_reason"] == "completed" assert seen["timeout_s"] == 1200 @pytest.mark.asyncio async def test_run_acp_case_converts_chat_exception_to_error_result(monkeypatch): case = load_case("component-restart-during-incident-window") calls: list[str] = [] async def _fake_run_chat_and_wait(**kwargs): raise RuntimeError("chat runner failed") async def _fake_release_acp_session(workspace_id: str) -> bool: calls.append(workspace_id) return True monkeypatch.setattr("investibook.api.chat.run_chat_and_wait", _fake_run_chat_and_wait) monkeypatch.setattr("investibook.core.agent_runtime.release_agent_session", _fake_release_acp_session) result = await _run_acp_case( prompt=_case_prompt(case), workspace_id="ws-eval-test", user_email="eval@localhost", timeout_s=120, idle_timeout_s=10, ) assert result["finish_reason"] == "error" assert result["error"] == "chat runner failed" assert calls == ["ws-eval-test"] def test_run_eval_suite_dry_run_supports_case_subset(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="dry-run", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", ) ) assert detail["suite_id"] == "core-daily-demo" assert detail["execution_status"] == "completed" assert detail["comparison_status"] == "review_only" assert detail["case_count"] == 1 assert [item["case_id"] for item in detail["case_runs"]] == ["service-degradation-during-incident-window"] assert detail["summary"]["selected_case_ids"] == ["service-degradation-during-incident-window"] assert "Evaluation Run Report" in detail["text_report"] assert "service-degradation-during-incident-window" in detail["text_report"] assert (tmp_path / "audit" / detail["id"] / "report.txt").exists() assert (tmp_path / "audit" / detail["id"] / "report_detail.txt").exists() def test_run_eval_suite_dry_run_supports_evaluation_mode_override(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) original_suite = load_suite("core-daily-demo") original_case = load_case("service-degradation-during-incident-window") detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="dry-run", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", evaluation_mode_override="evidence_only", ) ) case_run = detail["case_runs"][0] assert detail["suite_version"] != original_suite.suite_version assert detail["summary"]["runtime_metadata"]["evaluation_mode_override"] == "evidence_only" assert case_run["case_version"] != original_case.case_version assert case_run["summary"]["evaluation_mode"] == "evidence_only" assert case_run["summary"]["runtime_metadata"]["evaluation_mode_override"] == "evidence_only" assert "this case is evidence_only" in case_run["output"]["prompt"] def test_run_eval_suite_dry_run_records_repo_baseline_comparison(tmp_path, monkeypatch): registry_root = tmp_path / "regression" shutil.copytree( Path(__file__).resolve().parents[2] / "evaluation" / "regression", registry_root, ) suite = load_suite("core-daily-demo", registry_root=registry_root) (registry_root / "baselines").mkdir(exist_ok=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( f""" schema_version: 1 baseline_id: baseline-evalrun-old suite_id: core-daily-demo suite_version: {suite.suite_version} promoted_from_run_id: evalrun-old registry_revision: git-old created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: openai/gpt-5.2 summary: execution_status: completed comparison_status: pending cases: - case_id: service-degradation-during-incident-window case_version: case-old case_run_id: evalcase-old workspace_id: ws-old execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) db_path = tmp_path / "investibook.db" monkeypatch.setattr("investibook.core.eval.eval_runner.cfg.AGENT_TYPE", "opencode") monkeypatch.setattr("investibook.core.eval.eval_runner.cfg.ACP_MODEL_STANDARD", "openai/gpt-5.2") monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="dry-run", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", registry_root=registry_root, ) ) comparison = detail["summary"]["baseline_comparison"] assert comparison["baseline_id"] == "baseline-evalrun-old" assert comparison["regressed_count"] == 1 assert comparison["improved_count"] == 0 assert "Baseline comparison: source=repo baseline=baseline-evalrun-old regressed=1" in detail["text_report"] assert "For bucket-level and case-level details, see report_detail.txt." in detail["text_report"] assert "baseline diff: status=regressed baseline=pass/100 current=completed/-" in detail["text_report_detail"] def test_db_get_eval_run_detail_loads_baseline_export_from_registry_root(tmp_path, monkeypatch): app_root = tmp_path / "app" registry_root = app_root / "evaluation" / "regression" baseline_run_dir = registry_root / "baselines" / "runs" / "evalrun-base" case_dir = baseline_run_dir / "cases" / "case-a" case_dir.mkdir(parents=True, exist_ok=True) invalid_case_dir = baseline_run_dir / "cases" / "case-b" invalid_case_dir.mkdir(parents=True, exist_ok=True) (registry_root / "baselines").mkdir(parents=True, exist_ok=True) (registry_root / "baselines" / "core-daily-demo.yaml").write_text( """ schema_version: 1 baseline_id: baseline-1 suite_id: core-daily-demo suite_version: suite-v1 promoted_from_run_id: evalrun-base registry_revision: git-old created_at: 2026-05-11T00:00:00+00:00 created_by: tester notes: "" runtime: agent_type: opencode resolved_model: chat/claude-standard summary: execution_status: completed comparison_status: review_only runtime_export: dir: /evaluation/regression/baselines/runs/evalrun-base cases: - case_id: case-a case_version: case-v1 case_run_id: evalcase-base workspace_id: ws-base execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 100 passed_checks: 5 total_checks: 5 - case_id: case-b case_version: case-v1 case_run_id: evalcase-base-invalid workspace_id: ws-base-invalid execution_status: completed comparison_status: review_only judgment_verdict: pass primary_failure_code: "" score: 80 passed_checks: 4 total_checks: 5 """.strip() + "\n", encoding="utf-8", ) (case_dir / "case_result.yaml").write_text( """ score: 100 started_at: 2026-05-02T14:49:42.968960+00:00 finished_at: 2026-05-02T14:50:42.968960+00:00 token_usage_summary: session_count: 1 input_tokens: 100 output_tokens: 20 cache_read_tokens: 10 cache_write_tokens: 5 cost_amount: 0.25 """.strip() + "\n", encoding="utf-8", ) (case_dir / "judgment.json").write_text( """ {"judgment_verdict":"pass","packet":{"workflow":{"score":10,"max_score":10}}} """.strip() + "\n", encoding="utf-8", ) (invalid_case_dir / "case_result.yaml").write_text( """ score: 80 started_at: 2026-05-02T14:51:42.968960+00:00 finished_at: 2026-05-02T14:52:42.968960+00:00 token_usage_summary: session_count: 1 input_tokens: 40 output_tokens: 10 cache_read_tokens: 0 cache_write_tokens: 0 cost_amount: 0.10 """.strip() + "\n", encoding="utf-8", ) (invalid_case_dir / "judgment.json").write_text( """ {"judgment_verdict":"pass","packet":{"critical_evidence":{"score":20,"max_score":30},"diagnosis":{"score":15,"max_score":50}}} """.strip() + "\n", encoding="utf-8", ) db_path = tmp_path / "investibook.db" data_dir = tmp_path / "data" data_dir.mkdir() monkeypatch.setattr(dbq, "DB_PATH", db_path) monkeypatch.setattr(dbq, "DATA_DIR", data_dir) monkeypatch.setenv("EVAL_REGISTRY_ROOT", str(registry_root)) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-1", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-new", "trigger_mode": "scheduled", "execution_status": "completed", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "baseline_id": "baseline-1", "started_at": "2026-05-20T08:10:53.188289+00:00", "finished_at": "2026-05-20T10:02:26.000000+00:00", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": { "baseline_path": str(registry_root / "baselines" / "core-daily-demo.yaml"), "runtime_metadata": { "agent_type": "opencode", "resolved_model": "chat/claude-standard", }, }, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-1", "run_id": "evalrun-1", "case_id": "case-a", "case_version": "case-v1", "agent_revision": "git-new", "environment_fingerprint": "local:test", "judge_version": "judge-v1", "followup_policy_key": "", "execution_status": "completed", "comparison_status": "pending", "started_at": "2026-05-20T08:10:53.188289+00:00", "finished_at": "2026-05-20T08:11:53.188289+00:00", "output_text": "ok", "output": {}, "summary": {"workspace_id": "ws-1"}, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-2", "run_id": "evalrun-1", "case_id": "case-b", "case_version": "case-v1", "agent_revision": "git-new", "environment_fingerprint": "local:test", "judge_version": "judge-v1", "followup_policy_key": "", "execution_status": "invalid", "comparison_status": "pending", "started_at": "2026-05-20T08:12:53.188289+00:00", "finished_at": "2026-05-20T08:13:53.188289+00:00", "output_text": "timeout", "output": {}, "summary": {"workspace_id": "ws-2"}, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-1", "case_run_id": "evalcase-1", "judgment_verdict": "fail", "score": 70, "passed_checks": 1, "total_checks": 2, "facts": {}, "packet": { "workflow": {"score": 8, "max_score": 10}, }, "audit": {}, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-2", "case_run_id": "evalcase-2", "judgment_verdict": "invalid", "score": 0, "passed_checks": 0, "total_checks": 0, "facts": {}, "packet": {}, "audit": {}, } ) detail = dbq._db_get_eval_run_detail("evalrun-1") assert detail is not None assert detail["baseline_diff_metrics"]["compared_case_count"] == 2 case_comparisons = {case_run["case_id"]: case_run["baseline_comparison"] for case_run in detail["case_runs"]} baseline_comparison = case_comparisons["case-a"] assert baseline_comparison["compared"] is True assert baseline_comparison["baseline_score"] == 100.0 assert baseline_comparison["score_diff"] == -30.0 invalid_comparison = case_comparisons["case-b"] assert invalid_comparison["compared"] is True assert invalid_comparison["baseline_score"] == 80.0 assert invalid_comparison["score_diff"] == -80.0 assert invalid_comparison["current_bucket_scores"]["critical_evidence"]["score"] == 0.0 assert invalid_comparison["current_bucket_scores"]["diagnosis"]["score"] == 0.0 assert invalid_comparison["bucket_score_diffs"]["critical_evidence"] == -20.0 assert invalid_comparison["bucket_score_diffs"]["diagnosis"] == -15.0 def test_case_failure_detail_lines_prefers_human_readable_descriptions(): case_run = { "judgment": { "packet": { "workflow": { "status": "partial", "score": 5, "max_score": 10, "checks": [ { "id": "targeted_panic_log_query_completed", "description": "Targeted panic log query is completed.", "passed": False, "earned_score": 0, "max_score": 10, "snippet": "matched workflow facts: architecture_context_loaded, summary_read_completed", }, ], } } } } lines = _case_failure_detail_lines(case_run) assert lines == [ "workflow: partial score=5/10", " - targeted_panic_log_query_completed earned=0/10 desc=Targeted panic log query is completed. snippet=matched workflow facts: architecture_context_loaded, summary_read_completed", ] def test_format_eval_run_report_includes_total_score_duration_and_tokens(): report = _format_eval_run_report( { "id": "evalrun-1", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "execution_status": "completed", "comparison_status": "review_only", "trigger_mode": "manual_suite", "case_count": 3, "passed_count": 1, "failed_count": 1, "invalid_count": 1, "started_at": "2026-05-02T14:49:42.968960+00:00", "finished_at": "2026-05-02T14:53:27.968960+00:00", "token_usage_summary": { "workspace_count": 3, "session_count": 2, "input_tokens": 1200, "output_tokens": 340, "reasoning_tokens": 60, "cache_read_tokens": 80, "cache_write_tokens": 20, "cost_amount": 1.2345, }, "case_runs": [ { "case_id": "case-a", "workspace_id": "ws-1", "started_at": "2026-05-02T14:49:42.968960+00:00", "finished_at": "2026-05-02T14:50:42.968960+00:00", "token_usage_summary": { "session_count": 1, "input_tokens": 1200, "output_tokens": 340, "reasoning_tokens": 60, "cache_read_tokens": 80, "cache_write_tokens": 20, "cost_amount": 1.2345, }, "judgment": {"judgment_verdict": "pass", "score": 100}, }, { "case_id": "case-b", "workspace_id": "ws-2", "started_at": "2026-05-02T14:50:42.968960+00:00", "finished_at": "2026-05-02T14:52:12.968960+00:00", "token_usage_summary": { "session_count": 1, "input_tokens": 0, "output_tokens": 0, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cost_amount": 0.0, }, "judgment": {"judgment_verdict": "fail", "score": 70}, }, { "case_id": "case-c", "workspace_id": "ws-3", "started_at": "2026-05-02T14:52:12.968960+00:00", "finished_at": "2026-05-02T14:53:27.968960+00:00", "token_usage_summary": { "session_count": 0, "input_tokens": 0, "output_tokens": 0, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cost_amount": 0.0, }, "judgment": {"judgment_verdict": "invalid", "primary_failure_code": "acp_error"}, }, ], } ) assert "Score: avg=85 stddev=15 scored_cases=2/3" in report assert "Timing: avg=1m15s stddev=0m12s measured_cases=3/3" in report assert ( "Token usage: input_avg=400 input_stddev=565.69 " "output_avg=113.33 output_stddev=160.28 " "reasoning_avg=20 reasoning_stddev=28.28 " "cache_read_avg=26.67 cache_read_stddev=37.71 " "cache_write_avg=6.67 cache_write_stddev=9.43 " "cached_total_avg=33.33 cached_total_stddev=47.14 " "cost_avg=$0.4115 cost_stddev=$0.5819 measured_cases=3/3" ) in report def test_compute_baseline_comparison_detects_regressions_improvements_and_missing(): comparison = _compute_baseline_comparison( { "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", } }, "case_runs": [ { "case_id": "case-a", "execution_status": "completed", "judgment": { "judgment_verdict": "fail", "score": 70, "packet": { "workflow": {"status": "pass", "score": 8, "max_score": 10}, "diagnosis": {"status": "fail", "score": 35, "max_score": 50}, }, }, }, {"case_id": "case-b", "execution_status": "completed", "judgment": {"judgment_verdict": "pass", "score": 95}}, {"case_id": "case-c", "execution_status": "invalid", "judgment": {"judgment_verdict": "invalid", "score": 0}}, ] }, { "source": "repo", "baseline_id": "baseline-1", "path": "/tmp/baseline.yaml", "runtime_export": {"dir": "/tmp/does-not-exist"}, "runtime": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, "cases": [ { "case_id": "case-a", "judgment_verdict": "pass", "score": 100, "packet": { "workflow": {"status": "pass", "score": 10, "max_score": 10}, "diagnosis": {"status": "pass", "score": 45, "max_score": 50}, }, }, {"case_id": "case-b", "judgment_verdict": "fail", "score": 60}, {"case_id": "case-d", "judgment_verdict": "pass", "score": 90}, ], }, ) assert comparison is not None assert comparison["regressed_count"] == 1 assert comparison["improved_count"] == 1 assert comparison["new_case_count"] == 1 assert comparison["missing_case_count"] == 1 case_a = next(item for item in comparison["case_diffs"] if item["case_id"] == "case-a") assert [item["bucket"] for item in case_a["bucket_changes"]] == ["workflow", "diagnosis"] assert comparison["bucket_totals"][0]["bucket"] == "workflow" def test_compute_baseline_comparison_skips_when_runtime_metadata_mismatch(): comparison = _compute_baseline_comparison( { "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", } }, "case_runs": [ {"case_id": "case-a", "execution_status": "completed", "judgment": {"judgment_verdict": "pass", "score": 100}}, ], }, { "source": "repo", "baseline_id": "baseline-1", "path": "/tmp/baseline.yaml", "runtime": { "agent_type": "claude", "resolved_model": "anthropic/claude-sonnet", }, "cases": [ {"case_id": "case-a", "judgment_verdict": "pass", "score": 100}, ], }, ) assert comparison is not None assert comparison["status"] == "skipped" assert comparison["reason"] == "resolved_model_mismatch" def test_compute_baseline_comparison_allows_claude_family_runtime_variants(): comparison = _compute_baseline_comparison( { "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "chat/claude-standard/high", } }, "case_runs": [ {"case_id": "case-a", "execution_status": "completed", "judgment": {"judgment_verdict": "pass", "score": 100}}, ], }, { "source": "repo", "baseline_id": "baseline-1", "path": "/tmp/baseline.yaml", "runtime": { "agent_type": "claude", "resolved_model": "anthropic/claude-sonnet", }, "cases": [ {"case_id": "case-a", "judgment_verdict": "pass", "score": 100}, ], }, ) assert comparison is not None assert comparison["status"] == "compared" assert comparison["reason"] == "" assert comparison["unchanged_count"] == 1 def test_compute_baseline_comparison_allows_gpt_family_runtime_variants(): comparison = _compute_baseline_comparison( { "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.5", } }, "case_runs": [ {"case_id": "case-a", "execution_status": "completed", "judgment": {"judgment_verdict": "pass", "score": 100}}, ], }, { "source": "repo", "baseline_id": "baseline-1", "path": "/tmp/baseline.yaml", "runtime": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", }, "cases": [ {"case_id": "case-a", "judgment_verdict": "pass", "score": 100}, ], }, ) assert comparison is not None assert comparison["status"] == "compared" assert comparison["reason"] == "" assert comparison["unchanged_count"] == 1 def test_format_eval_run_report_shows_diagnosis_penalties(): report = _format_eval_run_report( { "id": "evalrun-penalty", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "execution_status": "completed", "comparison_status": "review_only", "trigger_mode": "manual_case", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:01:00+00:00", "token_usage_summary": {}, "case_runs": [ { "case_id": "component-restart-during-incident-window", "workspace_id": "ws-1", "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:00:10+00:00", "token_usage_summary": { "session_count": 0, "input_tokens": 0, "output_tokens": 0, "reasoning_tokens": 0, "cache_read_tokens": 0, "cache_write_tokens": 0, "cost_amount": 0.0, }, "judgment": { "judgment_verdict": "pass", "score": 70, "packet": { "diagnosis": { "status": "pass", "checks": [ { "id": "reject_missing_index_root_cause", "scoring_mode": "penalty", "penalty_applied": 3.0, } ], } }, }, } ], } ) assert "penalties=reject_missing_index_root_cause:-3" in report assert "Score: avg=70 scored_cases=1/1" in report assert "stddev=" not in report assert "Timing: avg=0m10s measured_cases=1/1" in report assert "Token usage: input_avg=0 output_avg=0 reasoning_avg=0 cache_read_avg=0 cache_write_avg=0 cached_total_avg=0 cost_avg=$0.0000 measured_cases=1/1" in report def test_format_eval_run_report_shows_bucket_totals_vs_baseline(): report = _format_eval_run_report( { "id": "evalrun-buckets", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "execution_status": "completed", "comparison_status": "review_only", "trigger_mode": "manual_suite", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:01:00+00:00", "token_usage_summary": {}, "summary": { "baseline_comparison": { "source": "repo", "baseline_id": "baseline-1", "status": "compared", "regressed_count": 1, "improved_count": 0, "unchanged_count": 0, "new_case_count": 0, "missing_case_count": 0, "bucket_totals": [ { "bucket": "workflow", "baseline_score": 10, "baseline_max_score": 10, "current_score": 5, "current_max_score": 10, "baseline_avg_score": 10, "baseline_avg_max_score": 10, "current_avg_score": 5, "current_avg_max_score": 10, "delta_avg_score": -5, "compared_case_count": 1, "delta_score": -5, "changed_case_count": 1, } ], "case_diffs": [], } }, "case_runs": [], } ) assert "Bucket Totals vs Baseline" in report assert "workflow: baseline_avg=10/10 current_avg=5/10 delta_avg=-5 compared_cases=1 changed_cases=1" in report def test_format_eval_run_detail_report_keeps_bucket_level_details(): report = _format_eval_run_detail_report( { "id": "evalrun-detail", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "execution_status": "completed", "comparison_status": "compared", "trigger_mode": "manual_suite", "schedule_date": "2026-05-09", "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:01:00+00:00", "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", "promote_eligibility": "eligible", "baseline_id": "baseline-1", "case_count": 1, "passed_count": 0, "failed_count": 1, "invalid_count": 0, "case_runs": [ { "case_id": "component-restart-during-incident-window", "workspace_id": "ws-1", "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:00:10+00:00", "execution_status": "completed", "output_text": "first line\nsecond line", "token_usage_summary": { "session_count": 1, "input_tokens": 800, "output_tokens": 220, "reasoning_tokens": 44, "cache_read_tokens": 40, "cache_write_tokens": 0, "cost_amount": 0.1234, }, "judgment": { "judgment_verdict": "fail", "score": 20, "primary_failure_code": "critical_evidence_missing", "packet": { "contract": {"status": "pass"}, "execution_contract": {"status": "pass"}, "workflow": {"status": "pass", "score": 10, "max_score": 10}, "evidence_collection": {"status": "pass", "score": 5, "max_score": 5}, "diagnosis": { "status": "fail", "score": 20, "max_score": 50, "checks": [ { "id": "reject_missing_index_root_cause", "scoring_mode": "penalty", "penalty_applied": 3.0, } ], }, "critical_evidence": { "status": "fail", "score": 0, "max_score": 30, "checks": [ { "id": "restart_signal_present", "description": "Direct restart evidence is present within the incident window.", "passed": False, "earned_score": 0, "max_score": 8, } ], }, "evidence_routing": {"status": "pass", "score": 5, "max_score": 5}, }, }, } ], } ) assert "Evaluation Run Detail Report" in report assert "Run Metadata" in report assert "Runtime Metadata" in report assert "Key Metrics" in report assert "Case: component-restart-during-incident-window" in report assert "diagnosis penalties: reject_missing_index_root_cause:-3" in report assert "token usage: input=800 output=220 reasoning=44 cache_read=40 cache_write=0 cached_total=40 cost=$0.1234" in report assert "Bucket Failure Details" in report assert "critical evidence: fail score=0/30" in report assert "diagnosis: fail score=20/50" in report assert ( "restart_signal_present earned=0/8 desc=Direct restart evidence is present within the incident window." in report ) assert "Raw Output" in report assert "first line" in report assert "second line" in report def test_format_eval_run_detail_report_shows_bucket_diff_against_baseline(): report = _format_eval_run_detail_report( { "id": "evalrun-detail-baseline", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "execution_status": "completed", "comparison_status": "compared", "trigger_mode": "manual_suite", "case_count": 1, "passed_count": 1, "failed_count": 0, "invalid_count": 0, "summary": { "baseline_comparison": { "source": "repo", "baseline_id": "baseline-1", "baseline_path": "/tmp/baseline.yaml", "status": "compared", "regressed_count": 1, "improved_count": 0, "unchanged_count": 0, "new_case_count": 0, "missing_case_count": 0, "case_diffs": [ { "case_id": "component-restart-during-incident-window", "status": "regressed", "baseline_verdict": "pass", "baseline_score": 71, "current_verdict": "pass", "current_score": 70, "baseline_token_usage": { "input_tokens": 1000, "output_tokens": 250, "reasoning_tokens": 30, "cache_read_tokens": 70, "cache_write_tokens": 0, "cached_total_tokens": 70, "cost_amount": 0.2222, }, "current_token_usage": { "input_tokens": 900, "output_tokens": 200, "reasoning_tokens": 20, "cache_read_tokens": 50, "cache_write_tokens": 10, "cached_total_tokens": 60, "cost_amount": 0.1111, }, "bucket_changes": [ { "bucket": "workflow", "baseline": {"status": "pass", "score": 10, "max_score": 10}, "current": {"status": "partial", "score": 5, "max_score": 10}, }, { "bucket": "diagnosis", "baseline": {"status": "pass", "score": 45, "max_score": 50}, "current": {"status": "pass", "score": 44, "max_score": 50}, }, ], "check_changes": [ { "bucket": "workflow", "check_id": "targeted_panic_log_query_completed", "baseline": {"passed": True, "earned_score": 10, "max_score": 10}, "current": {"passed": False, "earned_score": 0, "max_score": 10}, } ], } ], "missing_cases": [], } }, "case_runs": [ { "case_id": "component-restart-during-incident-window", "workspace_id": "ws-1", "started_at": "2026-05-09T10:00:00+00:00", "finished_at": "2026-05-09T10:00:10+00:00", "execution_status": "completed", "baseline_comparison": { "baseline_score": 71, "score_diff": -1, "current_bucket_scores": { "workflow": {"score": 5, "max_score": 10}, "diagnosis": {"score": 44, "max_score": 50}, }, "baseline_bucket_scores": { "workflow": {"score": 10, "max_score": 10}, "diagnosis": {"score": 45, "max_score": 50}, }, "bucket_score_diffs": { "workflow": -5, "diagnosis": -1, }, "influential_checks": [ { "bucket": "workflow", "check_id": "targeted_panic_log_query_completed", "description": "Targeted panic log query was not completed.", "impact_score": 10, "snippet": "panic log query missing", "semantic_notes": ["missed primary check"], } ], }, "token_usage_summary": { "session_count": 1, "input_tokens": 900, "output_tokens": 200, "reasoning_tokens": 20, "cache_read_tokens": 50, "cache_write_tokens": 10, "cost_amount": 0.1111, }, "judgment": { "judgment_verdict": "pass", "score": 70, "packet": { "workflow": {"status": "partial", "score": 5, "max_score": 10}, "diagnosis": {"status": "pass", "score": 44, "max_score": 50}, }, }, } ], } ) assert "Baseline Summary" in report assert "Bucket Totals vs Baseline" not in report assert "Bucket Scores vs Baseline" in report assert "workflow: current=5/10 baseline=10/10 diff=-5" in report assert "diagnosis: current=44/50 baseline=45/50 diff=-1" in report assert "Judging Conditions" in report assert "impact=10 desc=Targeted panic log query was not completed." in report assert "token usage: input=900 output=200 reasoning=20 cache_read=50 cache_write=10 cached_total=60 cost=$0.1111" in report assert "Baseline Diff" in report assert "token diff vs baseline:" in report assert "input=1000->900" in report assert "output=250->200" in report assert "reasoning=30->20" in report assert "cache_read=70->50" in report assert "cache_write=0->10" in report assert "cached_total=70->60" in report assert "cost=$0.2222->$0.1111" in report assert "workflow: pass score=10/10 -> partial score=5/10" in report assert "diagnosis: pass score=45/50 -> pass score=44/50" in report assert "check diff vs baseline:" in report assert "workflow.targeted_panic_log_query_completed: pass earned=10/10 -> fail earned=0/10" in report def test_run_eval_suite_rejects_unknown_case_id(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() with pytest.raises(ValueError, match="Unknown case id"): run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="dry-run", case_ids=("does-not-exist",), user_email="eval@localhost", ) ) def test_run_eval_suite_marks_run_completed_when_case_result_is_invalid(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) async def _fake_run_acp_case(*args, **kwargs): return { "prompt": "diagnose this", "text": "", "finish_reason": "timeout", "error": "Eval runner timeout after 30s", } monkeypatch.setattr("investibook.core.eval.eval_runner._run_acp_case", _fake_run_acp_case) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="acp", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", ) ) assert detail["execution_status"] == "completed" assert detail["invalid_count"] == 1 assert detail["case_runs"][0]["execution_status"] == "invalid" runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 1 assert runs[0]["execution_status"] == "completed" def test_run_eval_suite_ignores_unavailable_semantic_shadow(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) async def _fake_run_acp_case(*args, **kwargs): return { "prompt": "diagnose this", "text": PASSING_OUTPUT, "finish_reason": "completed", "error": None, } judge_calls: list[str] = [] def _fake_judge_case_output(*args, **kwargs): judge_calls.append("judge") return { "judgment_verdict": "pass", "score": 100, "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } monkeypatch.setattr("investibook.core.eval.eval_runner._run_acp_case", _fake_run_acp_case) monkeypatch.setattr("investibook.core.eval.eval_runner.judge_case_output", _fake_judge_case_output) monkeypatch.setattr( "investibook.core.eval.eval_runner.maybe_run_semantic_evaluator", lambda **kwargs: {"packet_status": "error", "shadow_available": False}, ) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="acp", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", ) ) assert detail["execution_status"] == "completed" assert judge_calls == ["judge"] def test_run_eval_suite_marks_run_invalid_when_interrupted(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) async def _fake_run_acp_case(*args, **kwargs): raise RuntimeError("boom") monkeypatch.setattr("investibook.core.eval.eval_runner._run_acp_case", _fake_run_acp_case) with pytest.raises(RuntimeError, match="boom"): run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", execution_mode="acp", case_ids=("service-degradation-during-incident-window",), user_email="eval@localhost", ) ) runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 1 assert runs[0]["execution_status"] == "invalid" assert runs[0]["summary"]["failure_reason"] == "runner_exception" detail = dbq._db_get_eval_run_detail(runs[0]["id"]) assert detail is not None assert detail["case_runs"][0]["execution_status"] == "invalid" def test_run_eval_suite_scheduled_mode_is_idempotent_by_suite_and_date(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-19", ) ) second = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-19", ) ) runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 1 assert detail["id"] == second["id"] assert runs[0]["schedule_date"] == "2026-05-19" assert runs[0]["idempotency_key"].startswith("scheduled:core-daily-demo:2026-05-19:opencode:") def test_run_eval_suite_scheduled_mode_separates_models_by_idempotency_key(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) monkeypatch.setattr( "investibook.core.eval.eval_runner.resolve_runtime_model_metadata", lambda **kwargs: {"agent_type": "opencode", "resolved_model": "openai/gpt-5.2"}, ) gpt_detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-21", ) ) monkeypatch.setattr( "investibook.core.eval.eval_runner.resolve_runtime_model_metadata", lambda **kwargs: {"agent_type": "claude", "resolved_model": "anthropic/claude-sonnet"}, ) claude_detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-21", ) ) runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 2 assert gpt_detail["id"] != claude_detail["id"] assert { run["idempotency_key"] for run in runs } == { "scheduled:core-daily-demo:2026-05-21:opencode:openai/gpt-5.2", "scheduled:core-daily-demo:2026-05-21:claude:anthropic/claude-sonnet", } def test_run_eval_suite_scheduled_releases_lease_on_completion(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-20", ) ) runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 1 assert runs[0]["lease_owner"] == "" assert runs[0]["job_exit_code"] == 0 def test_run_eval_suite_scheduled_mode_recovers_stale_same_idempotency_run(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) monkeypatch.setattr( "investibook.core.eval.eval_runner.resolve_runtime_model_metadata", lambda **kwargs: {"agent_type": "opencode", "resolved_model": "amazon-bedrock/claude-sonnet-bedrock"}, ) stale_key = "scheduled:core-daily-demo:2026-05-22:opencode:amazon-bedrock/claude-sonnet-bedrock" dbq._db_insert_eval_run( { "id": "evalrun-stale", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "running", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-22T00:00:00+00:00", "schedule_date": "2026-05-22", "idempotency_key": stale_key, "lease_owner": "evalrun-stale", "lease_until": "2000-01-01T00:30:00+00:00", "summary": { "execution_mode": "dry-run", "runtime_metadata": { "agent_type": "opencode", "resolved_model": "amazon-bedrock/claude-sonnet-bedrock", }, }, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-stale", "run_id": "evalrun-stale", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "running", "comparison_status": "pending", "started_at": "2026-05-22T00:01:00+00:00", "summary": { "workspace_id": "ws-stale", "execution_mode": "dry-run", }, } ) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-22", ) ) runs = dbq._db_get_eval_runs(limit=10) assert len(runs) == 2 assert detail["id"] != "evalrun-stale" latest_by_id = {run["id"]: run for run in runs} stale_run = latest_by_id["evalrun-stale"] assert stale_run["execution_status"] == "invalid" assert stale_run["lease_owner"] == "" assert stale_run["lease_until"] == "" assert stale_run["job_exit_code"] == 1 assert stale_run["idempotency_key"] == f"{stale_key}:stale:evalrun-stale" assert stale_run["summary"]["failure_reason"] == "stale_lease_expired" stale_case_run = dbq._db_get_eval_case_run("evalcase-stale") assert stale_case_run is not None assert stale_case_run["execution_status"] == "invalid" stale_judgment = dbq._db_get_eval_judgment_by_case_run("evalcase-stale") assert stale_judgment is not None assert stale_judgment["judgment_verdict"] == "invalid" assert stale_judgment["primary_failure_code"] == "stale_lease_expired" new_run = latest_by_id[detail["id"]] assert new_run["execution_status"] == "completed" assert new_run["idempotency_key"] == stale_key def test_run_eval_suite_scheduled_mode_ignores_other_stale_suite_run(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces") monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit") monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None) monkeypatch.setattr("investibook.core.eval.eval_runner._write_eval_workspace_env", lambda *args, **kwargs: None) monkeypatch.setattr( "investibook.core.eval.eval_runner.resolve_runtime_model_metadata", lambda **kwargs: {"agent_type": "opencode", "resolved_model": "chat/claude-standard"}, ) dbq._db_insert_eval_run( { "id": "evalrun-other-stale", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "running", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-21T00:00:00+00:00", "schedule_date": "2026-05-21", "idempotency_key": "scheduled:core-daily-demo:2026-05-21:opencode:amazon-bedrock/claude-sonnet-bedrock", "lease_owner": "evalrun-other-stale", "lease_until": "2000-01-01T00:30:00+00:00", "summary": { "execution_mode": "dry-run", "runtime_metadata": { "agent_type": "opencode", "resolved_model": "amazon-bedrock/claude-sonnet-bedrock", }, }, } ) detail = run_eval_suite( EvalRunnerOptions( suite_id="core-daily-demo", trigger_mode="scheduled", execution_mode="dry-run", schedule_date="2026-05-23", ) ) recovered_run = dbq._db_get_eval_run("evalrun-other-stale") assert recovered_run is not None assert recovered_run["execution_status"] == "invalid" assert recovered_run["idempotency_key"].endswith(":stale:evalrun-other-stale") assert detail["execution_status"] == "completed"