from pathlib import Path from fastapi import FastAPI from fastapi.testclient import TestClient from investibook.api import admin from investibook.api.deps import require_admin from investibook.core.eval.eval_runner import EvalRunnerOptions from investibook.db import queries as dbq def _build_client(*, raise_server_exceptions: bool = True): app = FastAPI() app.include_router(admin.router) app.dependency_overrides[require_admin] = lambda: {"id": "u-admin", "email": "admin@example.com"} return TestClient(app, raise_server_exceptions=raise_server_exceptions) def test_eval_run_list_and_detail_endpoints_include_case_judgment_and_usage(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-1", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-1", "trigger_mode": "manual_case", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-01T00:00:00+00:00", "case_count": 1, "summary": {"execution_mode": "dry-run", "selected_case_ids": ["component-restart-during-incident-window"]}, } ) dbq._db_insert_eval_case_run( { "id": "evalcase-1", "run_id": "evalrun-1", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-01T00:00:00+00:00", "finished_at": "2026-05-01T00:01:00+00:00", "output_text": "summary", "output": {"workspace_id": "ws-eval-1"}, "summary": {"workspace_id": "ws-eval-1", "finish_reason": "completed"}, } ) dbq._db_insert_eval_judgment( { "id": "evaljudge-1", "case_run_id": "evalcase-1", "judgment_verdict": "fail", "primary_failure_code": "diagnosis_critical_gate_failed", "score": 42, "passed_checks": 3, "total_checks": 5, "facts": {}, "packet": {}, "audit": {}, } ) dbq._db_insert_token_usage( workspace_id="ws-eval-1", session_id="session-1", model_tier="standard", channel="eval", user_email="admin@example.com", input_tokens=120, output_tokens=45, reasoning_tokens=9, cache_read_tokens=10, cache_write_tokens=0, cost_amount=0.12, cost_currency="USD", ) with _build_client() as client: list_res = client.get("/api/admin/eval-runs") assert list_res.status_code == 200 runs = list_res.json()["runs"] assert len(runs) == 1 assert runs[0]["id"] == "evalrun-1" assert runs[0]["score_summary"] == { "total_score": 42.0, "max_score": 100, "average_score": 42.0, "scored_case_count": 1, "case_count": 1, } detail_res = client.get("/api/admin/eval-runs/evalrun-1") assert detail_res.status_code == 200 run = detail_res.json()["run"] assert run["token_usage_summary"]["input_tokens"] == 120 assert run["token_usage_summary"]["reasoning_tokens"] == 9 assert run["case_runs"][0]["workspace_id"] == "ws-eval-1" assert run["case_runs"][0]["judgment"]["judgment_verdict"] == "fail" assert run["case_runs"][0]["token_usage_summary"]["reasoning_tokens"] == 9 assert run["case_runs"][0]["token_usage_summary"]["cost_amount"] == 0.12 def test_create_eval_run_endpoint_schedules_subset_run(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" captured: dict[str, object] = {} def _fake_schedule(options: EvalRunnerOptions) -> None: captured["options"] = options monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr(admin, "_schedule_eval_run", _fake_schedule) with _build_client() as client: res = client.post( "/api/admin/eval-runs", json={ "suite_id": "core-daily-demo", "execution_mode": "dry-run", "case_ids": ["service-degradation-during-incident-window"], }, ) assert res.status_code == 200 payload = res.json() assert payload["status"] == "started" assert payload["trigger_mode"] == "manual_case" assert payload["case_ids"] == ["service-degradation-during-incident-window"] options = captured["options"] assert isinstance(options, EvalRunnerOptions) assert options.case_ids == ("service-degradation-during-incident-window",) assert options.trigger_mode == "manual_case" assert dbq._db_get_eval_run(payload["run_id"]) is not None def test_create_eval_run_endpoint_defaults_to_acp(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" captured: dict[str, object] = {} def _fake_schedule(options: EvalRunnerOptions) -> None: captured["options"] = options monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() monkeypatch.setattr(admin, "_schedule_eval_run", _fake_schedule) with _build_client() as client: res = client.post( "/api/admin/eval-runs", json={ "suite_id": "core-daily-demo", }, ) assert res.status_code == 200 options = captured["options"] assert isinstance(options, EvalRunnerOptions) assert options.execution_mode == "acp" assert options.trigger_mode == "manual_suite" assert dbq._db_get_eval_run(res.json()["run_id"]) is not None def test_eval_run_summary_and_status_endpoints(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-running", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "running", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-19T00:00:00+00:00", "schedule_date": "2026-05-19", "idempotency_key": "scheduled:core-daily-demo:2026-05-19", "lease_owner": "evalrun-running", "lease_until": "2026-05-19T02:00:00+00:00", } ) dbq._db_insert_eval_run( { "id": "evalrun-completed", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "completed", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-18T00:00:00+00:00", "finished_at": "2026-05-18T00:05:00+00:00", "schedule_date": "2026-05-18", "idempotency_key": "scheduled:core-daily-demo:2026-05-18", "case_count": 2, "passed_count": 1, "failed_count": 1, "invalid_count": 0, } ) with _build_client() as client: summary_res = client.get("/api/admin/eval-runs/summary?suite_id=core-daily-demo&trigger_mode=scheduled") assert summary_res.status_code == 200 summary = summary_res.json() assert summary["total_runs"] == 2 assert summary["active_runs"] == 1 assert summary["passed_count"] == 1 assert summary["failed_count"] == 1 status_res = client.get("/api/admin/eval-runs/status?suite_id=core-daily-demo&trigger_mode=scheduled") assert status_res.status_code == 200 status = status_res.json() assert status["running"] is True assert status["active_run_count"] == 1 assert status["active_runs"][0]["id"] == "evalrun-running" completed_summary_res = client.get( "/api/admin/eval-runs/summary?suite_id=core-daily-demo&trigger_mode=scheduled&execution_status=completed" ) assert completed_summary_res.status_code == 200 completed_summary = completed_summary_res.json() assert completed_summary["total_runs"] == 1 assert completed_summary["active_runs"] == 0 assert completed_summary["completed_runs"] == 1 assert completed_summary["latest_run_id"] == "evalrun-completed" completed_status_res = client.get( "/api/admin/eval-runs/status?suite_id=core-daily-demo&trigger_mode=scheduled&execution_status=completed" ) assert completed_status_res.status_code == 200 completed_status = completed_status_res.json() assert completed_status["running"] is False assert completed_status["active_run_count"] == 0 assert completed_status["active_runs"] == [] def test_eval_run_endpoints_filter_by_runtime_metadata(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-gpt", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "completed", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-19T00:00:00+00:00", "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "openai/gpt-5.2", } }, } ) dbq._db_insert_eval_run( { "id": "evalrun-claude", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "scheduled", "execution_status": "running", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-20T00:00:00+00:00", "summary": { "runtime_metadata": { "agent_type": "claude", "resolved_model": "anthropic/claude-sonnet", } }, } ) dbq._db_insert_eval_run( { "id": "evalrun-claude-high", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "pending", "promote_eligibility": "not_evaluated", "started_at": "2026-05-21T00:00:00+00:00", "summary": { "runtime_metadata": { "agent_type": "opencode", "resolved_model": "chat/claude-standard/high", } }, } ) with _build_client() as client: list_res = client.get("/api/admin/eval-runs?agent_type=opencode&resolved_model=openai/gpt-5.2") assert list_res.status_code == 200 runs = list_res.json()["runs"] assert len(runs) == 1 assert runs[0]["id"] == "evalrun-gpt" assert runs[0]["agent_type"] == "opencode" assert runs[0]["resolved_model"] == "openai/gpt-5.2" summary_res = client.get("/api/admin/eval-runs/summary?agent_type=claude&resolved_model=anthropic/claude-sonnet") assert summary_res.status_code == 200 summary = summary_res.json() assert summary["total_runs"] == 1 assert summary["active_runs"] == 1 assert summary["latest_run_id"] == "evalrun-claude" status_res = client.get("/api/admin/eval-runs/status?agent_type=claude&resolved_model=anthropic/claude-sonnet") assert status_res.status_code == 200 status = status_res.json() assert status["running"] is True assert status["active_run_count"] == 1 assert status["active_runs"][0]["id"] == "evalrun-claude" family_list_res = client.get("/api/admin/eval-runs?model_family=claude") assert family_list_res.status_code == 200 family_runs = family_list_res.json()["runs"] assert [run["id"] for run in family_runs] == ["evalrun-claude-high", "evalrun-claude"] family_summary_res = client.get("/api/admin/eval-runs/summary?model_family=claude") assert family_summary_res.status_code == 200 family_summary = family_summary_res.json() assert family_summary["total_runs"] == 2 assert family_summary["active_runs"] == 1 assert family_summary["latest_run_id"] == "evalrun-claude-high" family_status_res = client.get("/api/admin/eval-runs/status?model_family=claude") assert family_status_res.status_code == 200 family_status = family_status_res.json() assert family_status["running"] is True assert family_status["active_run_count"] == 1 assert family_status["active_runs"][0]["id"] == "evalrun-claude" def test_create_eval_run_endpoint_rejects_unknown_suite(monkeypatch): monkeypatch.setattr(admin, "_schedule_eval_run", lambda options: None) with _build_client(raise_server_exceptions=False) as client: res = client.post( "/api/admin/eval-runs", json={ "suite_id": "does-not-exist", }, ) assert res.status_code == 404 assert "Missing registry file" in res.json()["detail"] def test_create_eval_run_endpoint_rejects_non_positive_timeout(monkeypatch): monkeypatch.setattr(admin, "_schedule_eval_run", lambda options: None) with _build_client() as client: res = client.post( "/api/admin/eval-runs", json={ "suite_id": "core-daily-demo", "timeout_s": 0, }, ) assert res.status_code == 422 def test_rerun_failed_endpoint_schedules_only_non_passing_cases(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-source", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-1", "trigger_mode": "manual_suite", "execution_status": "completed", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-01T00:00:00+00:00", "case_count": 2, "summary": { "execution_mode": "acp", "timeout_s": 333, "idle_timeout_s": 44, }, } ) for case_id in [ "component-restart-during-incident-window", "service-degradation-during-incident-window", ]: dbq._db_insert_eval_case_run( { "id": f"case-{case_id}", "run_id": "evalrun-source", "case_id": case_id, "case_version": "case-v1", "execution_status": "completed", "comparison_status": "review_only", "started_at": "2026-05-01T00:00:00+00:00", "summary": {}, } ) dbq._db_insert_eval_judgment( { "id": "judge-pass", "case_run_id": "case-component-restart-during-incident-window", "judgment_verdict": "pass", "passed_checks": 1, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) dbq._db_insert_eval_judgment( { "id": "judge-fail", "case_run_id": "case-service-degradation-during-incident-window", "judgment_verdict": "fail", "primary_failure_code": "diagnosis_critical_gate_failed", "passed_checks": 0, "total_checks": 1, "facts": {}, "packet": {}, "audit": {}, } ) captured: dict[str, object] = {} def _fake_schedule(options: EvalRunnerOptions) -> None: captured["options"] = options monkeypatch.setattr(admin, "_schedule_eval_run", _fake_schedule) with _build_client() as client: res = client.post("/api/admin/eval-runs/evalrun-source/rerun-failed", json={}) assert res.status_code == 200 payload = res.json() assert payload["status"] == "started" assert payload["case_ids"] == ["service-degradation-during-incident-window"] options = captured["options"] assert isinstance(options, EvalRunnerOptions) assert options.trigger_mode == "rerun_failed" assert options.execution_mode == "acp" assert options.timeout_s == 333 assert options.idle_timeout_s == 44 assert options.case_ids == ("service-degradation-during-incident-window",) assert dbq._db_get_eval_run(payload["run_id"]) is not None def test_rerun_failed_endpoint_rejects_running_source_run(tmp_path, monkeypatch): db_path = tmp_path / "investibook.db" monkeypatch.setattr(dbq, "DB_PATH", db_path) dbq._db_init() dbq._db_insert_eval_run( { "id": "evalrun-running", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "registry_revision": "git-1", "trigger_mode": "manual_suite", "execution_status": "running", "comparison_status": "review_only", "promote_eligibility": "not_evaluated", "started_at": "2026-05-01T00:00:00+00:00", "case_count": 1, "summary": {"execution_mode": "acp"}, } ) dbq._db_insert_eval_case_run( { "id": "case-running", "run_id": "evalrun-running", "case_id": "component-restart-during-incident-window", "case_version": "case-v1", "execution_status": "running", "comparison_status": "review_only", "started_at": "2026-05-01T00:00:00+00:00", "summary": {}, } ) monkeypatch.setattr(admin, "_schedule_eval_run", lambda options: None) with _build_client() as client: res = client.post("/api/admin/eval-runs/evalrun-running/rerun-failed", json={}) assert res.status_code == 409 assert "must be terminal" in res.json()["detail"]