import pytest

from investibook.db import queries as dbq
from investibook.core.eval.eval_runner import EvalRunnerOptions, run_eval_suite


def test_run_eval_suite_counts_judge_invalid_as_invalid(tmp_path, monkeypatch):
    db_path = tmp_path / "investibook.db"
    monkeypatch.setattr(dbq, "DB_PATH", db_path)
    dbq._db_init()
    monkeypatch.setattr("investibook.core.eval.eval_runner.WORKSPACES_DIR", tmp_path / "workspaces")
    monkeypatch.setattr("investibook.core.eval.eval_runner.EVAL_AUDIT_DIR", tmp_path / "audit")
    monkeypatch.setattr("investibook.core.eval.eval_runner._write_workspace_opencode_json", lambda *args, **kwargs: None)
    monkeypatch.setattr("investibook.core.eval.eval_runner._populate_workspace_dotenv_from_server", lambda *args, **kwargs: None)
    monkeypatch.setattr(
        "investibook.core.eval.eval_runner._mirror_eval_audit_artifacts",
        lambda *args, **kwargs: tmp_path / "audit" / "case",
    )

    async def _fake_run_acp_case(*args, **kwargs):
        return {
            "prompt": "diagnose this",
            "text": "completed output",
            "finish_reason": "completed",
            "error": None,
        }

    def _fake_judge_case_output(*args, **kwargs):
        return {
            "judgment_verdict": "invalid",
            "invalid_subtype": "workflow",
            "primary_failure_code": "workflow_incomplete",
            "score": 87,
            "passed_checks": 0,
            "total_checks": 0,
            "facts": {},
            "packet": {},
            "audit": {},
        }

    monkeypatch.setattr("investibook.core.eval.eval_runner._run_acp_case", _fake_run_acp_case)
    monkeypatch.setattr("investibook.core.eval.eval_runner.judge_case_output", _fake_judge_case_output)

    detail = run_eval_suite(
        EvalRunnerOptions(
            suite_id="core-daily-demo",
            execution_mode="acp",
            case_ids=("service-degradation-during-incident-window",),
            user_email="eval@localhost",
        )
    )

    assert detail["execution_status"] == "completed"
    assert detail["passed_count"] == 0
    assert detail["failed_count"] == 0
    assert detail["invalid_count"] == 1
    assert detail["case_runs"][0]["judgment"]["judgment_verdict"] == "invalid"