import json from pathlib import Path from investibook.core.eval.eval_cli import main from investibook.core.eval.eval_runner import EvalRunnerOptions def test_eval_cli_list_outputs_active_suite(capsys): exit_code = main(["list"]) captured = capsys.readouterr() assert exit_code == 0 assert "Available suites" in captured.out assert "core-daily-demo" in captured.out assert "[active]" in captured.out def test_eval_cli_list_json_outputs_machine_readable_payload(capsys): exit_code = main(["--list", "--json"]) captured = capsys.readouterr() payload = json.loads(captured.out) suites_by_id = {item["suite_id"]: item for item in payload} assert exit_code == 0 assert suites_by_id["core-daily-demo"]["status"] == "active" def test_eval_cli_run_forwards_options_and_prints_report(monkeypatch, capsys): seen: dict[str, object] = {} def _fake_run_eval_suite(options: EvalRunnerOptions) -> dict[str, object]: seen["options"] = options return { "id": "evalrun-test", "text_report": "Evaluation Run Report\nSuite: core-daily-demo", "text_report_path": "/tmp/evalrun-test/report.txt", } monkeypatch.setattr("investibook.core.eval.eval_cli.run_eval_suite", _fake_run_eval_suite) exit_code = main( [ "--suite", "core-daily-demo", "--execution-mode", "acp", "--evaluation-mode-override", "evidence_only", "--case-id", "service-degradation-during-incident-window", ] ) captured = capsys.readouterr() options = seen["options"] assert exit_code == 0 assert isinstance(options, EvalRunnerOptions) assert options.suite_id == "core-daily-demo" assert options.execution_mode == "acp" assert options.evaluation_mode_override == "evidence_only" assert options.trigger_mode == "manual_case" assert options.case_ids == ("service-degradation-during-incident-window",) assert "Evaluation Run Report" in captured.out assert "Report saved to: /tmp/evalrun-test/report.txt" in captured.out def test_eval_cli_run_defaults_to_acp(monkeypatch, capsys): seen: dict[str, object] = {} def _fake_run_eval_suite(options: EvalRunnerOptions) -> dict[str, object]: seen["options"] = options return { "id": "evalrun-test", "text_report": "Evaluation Run Report\nSuite: core-daily-demo", "text_report_path": "/tmp/evalrun-test/report.txt", } monkeypatch.setattr("investibook.core.eval.eval_cli.run_eval_suite", _fake_run_eval_suite) exit_code = main(["--suite", "core-daily-demo"]) capsys.readouterr() options = seen["options"] assert exit_code == 0 assert isinstance(options, EvalRunnerOptions) assert options.execution_mode == "acp" def test_eval_cli_promote_writes_manifest_summary(monkeypatch, capsys): manifest = { "baseline_id": "baseline-evalrun-test", "promoted_from_run_id": "evalrun-test", "suite_id": "core-daily-demo", "suite_version": "suite-v1", "path": str(Path("/tmp/baselines/core-daily-demo.yaml")), } monkeypatch.setattr( "investibook.core.eval.eval_cli.promote_run_to_repo_baseline", lambda *args, **kwargs: manifest, ) exit_code = main(["promote", "--run-id", "evalrun-test"]) captured = capsys.readouterr() assert exit_code == 0 assert "Promoted run: evalrun-test" in captured.out assert "Manifest saved to: /tmp/baselines/core-daily-demo.yaml" in captured.out def test_eval_cli_rejudge_baseline_case_prints_summary(monkeypatch, capsys): monkeypatch.setattr( "investibook.core.eval.eval_cli.rejudge_exported_baseline_case", lambda **kwargs: { "run_id": "evalrun-1", "case_id": "component-restart-during-incident-window", "case_dir": "/tmp/baselines/runs/evalrun-1/cases/component-restart-during-incident-window", "judgment": { "judgment_verdict": "pass", "score": 100, "primary_failure_code": None, }, }, ) exit_code = main( [ "rejudge-baseline-case", "--run-id", "evalrun-1", "--case-id", "component-restart-during-incident-window", ] ) captured = capsys.readouterr() assert exit_code == 0 assert "Rejudged case: component-restart-during-incident-window" in captured.out assert "Verdict: pass score=100 failure=-" in captured.out