"""Findings from the post-merge review of the four assistant PRs.

Each of these was a number the harness would have reported wrongly, or a
refusal its own docstring promised and did not perform. They are grouped here
rather than spread through the suites they belong to because they share a
cause: a measurement that looks right until you ask what its denominator is.
"""
from __future__ import annotations

import uuid

import pytest

pytestmark = pytest.mark.asyncio


class _Judge:
    """Stand-in for a judge reply — same attributes ``combine_verdict`` reads."""

    def __init__(self, scores: dict[str, int | None], verdict: str = "pass") -> None:
        self.available = True
        self.scores = {k: v for k, v in scores.items() if v is not None}
        self.verdict = verdict
        self.failure_type = None


# ── citations ───────────────────────────────────────────────────────


def test_the_portal_link_the_prompt_demands_is_not_a_fabrication() -> None:
    """🔴 The prompt ENDS by requiring this link, and the check called it made up.

    ``ungrounded_citation`` is a BLOCKING check: it fails the attempt outright,
    past the judge, and files it under the wrong cause. No tool result contains
    the portal URL, so every correctly-behaved answer that pointed the employee
    somewhere was scored as a fabricated citation — a systematic depression of
    the pass rate produced by the assistant doing exactly what it was told.
    """
    from app.assistant.evaluation.checks import citation_findings
    from app.assistant.reply_hygiene import PORTAL, PORTAL_REQUESTS

    assert citation_findings(f"你可以在 Helix 提工单:{PORTAL_REQUESTS}", []) == []
    assert citation_findings(f"你的设备在 {PORTAL} 上", []) == []


def test_a_made_up_path_on_the_right_host_is_still_a_fabrication() -> None:
    """The allowlist is exact URLs, not a host — the tight version is the point.

    Without this the fix would have opened a hole the size of the domain: a
    model inventing ``helix.pingcap.net/assets/12345`` produces a link that
    404s, and an employee cannot tell that from a real one.
    """
    from app.assistant.evaluation.checks import citation_findings

    found = citation_findings("见 https://helix.pingcap.net/assets/12345", [])
    assert len(found) == 1
    found_elsewhere = citation_findings("见 https://evil.example.com/x", [])
    assert len(found_elsewhere) == 1


# ── the judge's floor ───────────────────────────────────────────────


def test_a_hard_dimension_the_judge_skipped_is_an_error_not_a_pass() -> None:
    """🔴 The floor was bypassable by the judge simply not answering.

    Its whole purpose is that 「pass」 cannot mean 「the model said pass while
    scoring groundedness 1」. The "no usable scores" branch only fires when
    EVERY dimension is missing, so a reply scoring the soft four and nulling
    both hard ones sailed through with a recorded verdict of pass.
    """
    from app.assistant.evaluation.runner import combine_verdict

    verdict, failure = combine_verdict(
        findings=[],
        judge=_Judge({"grounded": None, "accurate": None, "complete": 4, "tone": 5}),
    )
    assert verdict == "error"
    assert failure == "judge_error"


def test_a_soft_dimension_the_judge_skipped_is_still_a_pass() -> None:
    """The sentinel is right for the SOFT dimensions and must stay that way."""
    from app.assistant.evaluation.runner import combine_verdict

    verdict, _ = combine_verdict(
        findings=[], judge=_Judge({"grounded": 5, "accurate": 4}),
    )
    assert verdict == "pass"


def test_a_floored_hard_dimension_still_fails() -> None:
    from app.assistant.evaluation.runner import combine_verdict

    verdict, failure = combine_verdict(
        findings=[], judge=_Judge({"grounded": 1, "accurate": 4}),
    )
    assert verdict == "fail"
    assert failure == "tool_field_or_data"


# ── comparing two runs ──────────────────────────────────────────────


async def _run(db, *, suite_id, status="succeeded", **columns):
    from app.assistant.evaluation.models import AssistantEvalRun

    row = AssistantEvalRun(
        id=uuid.uuid4(), suite_id=suite_id, status=status,
        trigger="manual", repeats=1, **columns,
    )
    db.add(row)
    await db.flush()
    return row


async def _case(db, *, suite_id, question="问题"):
    """A real case row. ``assistant_eval_results.case_id`` has an FK to it —
    the results are deliberately not orphanable, so a test cannot fake one."""
    from app.assistant.evaluation.models import AssistantEvalCase

    row = AssistantEvalCase(
        id=uuid.uuid4(), suite_id=suite_id, question=question, category="kb_lookup",
    )
    db.add(row)
    await db.flush()
    return row


async def _result(db, *, run, case_id, verdict, attempt=1, question="问题"):
    from app.assistant.evaluation.models import AssistantEvalResult

    db.add(
        AssistantEvalResult(
            id=uuid.uuid4(), run_id=run.id, case_id=case_id, attempt=attempt,
            question=question, category="kb_lookup", verdict=verdict, reply="回答",
        )
    )
    await db.flush()


async def _suite(db):
    from app.assistant.evaluation.models import AssistantEvalSuite

    row = AssistantEvalSuite(
        id=uuid.uuid4(), slug=f"s-{uuid.uuid4().hex[:8]}", name="套件",
    )
    db.add(row)
    await db.flush()
    return row


async def test_a_truncated_run_is_not_compared_against_a_complete_one(db_session):
    """🔴 Selection bias, not a regression.

    A run whose budget runs out covers a PREFIX of the suite — attempts are
    ordered case-major on purpose — and a partial run is both alertable and
    eligible as a baseline. Comparing its headline against a complete run's
    reads the hardest eight cases as a 45-point drop with nothing changed.
    """
    from app.assistant.evaluation.service import compare_runs

    suite = await _suite(db_session)
    hard = [(await _case(db_session, suite_id=suite.id)).id for _ in range(4)]
    easy = [(await _case(db_session, suite_id=suite.id)).id for _ in range(20)]

    baseline = await _run(db_session, suite_id=suite.id, pass_rate=85.0)
    for case_id in hard:
        for attempt in range(5):
            await _result(
                db_session, run=baseline, case_id=case_id,
                verdict="fail", attempt=attempt,
            )
    for case_id in easy:
        for attempt in range(5):
            await _result(
                db_session, run=baseline, case_id=case_id,
                verdict="pass", attempt=attempt,
            )

    # The truncated run reached ONLY the hard cases, and did exactly as well on
    # them as the baseline did.
    truncated = await _run(db_session, suite_id=suite.id, status="partial", pass_rate=0.0)
    for case_id in hard:
        for attempt in range(5):
            await _result(
                db_session, run=truncated, case_id=case_id,
                verdict="fail", attempt=attempt,
            )
    await db_session.commit()

    diff = await compare_runs(db_session, run=truncated, baseline=baseline)
    assert diff["shared_cases"] == 4
    assert diff["drop_pp"] == 0.0, (
        "the headline comparison read a prefix of the suite as a regression"
    )


async def test_an_outage_is_not_comparable_on_two_surviving_samples(db_session):
    """🔴 The floor counted errored attempts, which pass_rate deliberately excludes.

    28 of 30 attempts erroring left two samples deciding a 45-point "drop",
    during exactly the outage the error exclusion exists to keep off the trend.
    """
    from app.assistant.evaluation.service import compare_runs

    suite = await _suite(db_session)
    cases = [(await _case(db_session, suite_id=suite.id)).id for _ in range(30)]

    baseline = await _run(db_session, suite_id=suite.id, pass_rate=95.0)
    for case_id in cases:
        await _result(db_session, run=baseline, case_id=case_id, verdict="pass")

    outage = await _run(db_session, suite_id=suite.id, pass_rate=50.0)
    await _result(db_session, run=outage, case_id=cases[0], verdict="pass")
    await _result(db_session, run=outage, case_id=cases[1], verdict="fail")
    for case_id in cases[2:]:
        await _result(db_session, run=outage, case_id=case_id, verdict="error")
    await db_session.commit()

    diff = await compare_runs(db_session, run=outage, baseline=baseline)
    assert diff["shared_scored"] == 2
    assert diff["comparable"] is False, (
        "two surviving samples were treated as a comparable measurement"
    )


async def test_a_real_regression_on_a_full_run_is_still_comparable(db_session):
    """The gate must not be so tight that nothing ever alerts."""
    from app.assistant.evaluation.service import compare_runs

    suite = await _suite(db_session)
    cases = [(await _case(db_session, suite_id=suite.id)).id for _ in range(25)]

    baseline = await _run(db_session, suite_id=suite.id, pass_rate=100.0)
    for case_id in cases:
        await _result(db_session, run=baseline, case_id=case_id, verdict="pass")
    now = await _run(db_session, suite_id=suite.id, pass_rate=40.0)
    for index, case_id in enumerate(cases):
        await _result(
            db_session, run=now, case_id=case_id,
            verdict="pass" if index < 10 else "fail",
        )
    await db_session.commit()

    diff = await compare_runs(db_session, run=now, baseline=baseline)
    assert diff["comparable"] is True
    assert diff["drop_pp"] == 60.0


# ── ordering ────────────────────────────────────────────────────────


async def test_the_results_table_puts_failures_on_the_first_screen(db_session):
    """🔴 The comment said "failures first" and the ORDER BY said the opposite.

    ``pass`` > ``fail`` > ``error`` in string order, so descending gave exactly
    the thirty passes the comment says it is preventing.
    """
    from app.assistant.evaluation.service import list_results

    suite = await _suite(db_session)
    run = await _run(db_session, suite_id=suite.id)
    for _ in range(5):
        case = await _case(db_session, suite_id=suite.id)
        await _result(db_session, run=run, case_id=case.id, verdict="pass")
    failing = await _case(db_session, suite_id=suite.id)
    await _result(db_session, run=run, case_id=failing.id, verdict="fail")
    await db_session.commit()

    rows, _total = await list_results(db_session, run_id=run.id, page=1, page_size=3)
    assert rows[0]["verdict"] == "fail", [r["verdict"] for r in rows]


# ── a run that measured nothing ─────────────────────────────────────


async def test_a_persona_who_has_left_is_refused(db_session):
    """🔴 The docstring promised this refusal; only the FALLBACK performed it.

    A run under a leaver measures the leaver gate, not the assistant — and puts
    a departed employee's data into a row an auditor can read.
    """
    from app.assistant.evaluation.models import AssistantEvalCase
    from app.assistant.evaluation.runner import EvalRunError, resolve_persona
    from app.identity.person_models import IdentityPerson

    leaver = IdentityPerson(
        id=uuid.uuid4(), display_name="已离职的人",
        work_email=f"left.{uuid.uuid4().hex[:8]}@test.com",
        employment_status="left", source_system="successfactors",
        hris_person_id=f"sf-{uuid.uuid4().hex[:10]}",
    )
    db_session.add(leaver)
    suite = await _suite(db_session)
    case = AssistantEvalCase(
        id=uuid.uuid4(), suite_id=suite.id, question="我的电脑呢",
        category="self_data", persona_person_id=leaver.id,
    )
    db_session.add(case)
    await db_session.commit()

    with pytest.raises(EvalRunError) as caught:
        await resolve_persona(db_session, case=case, suite=suite)
    assert caught.value.code == "ASSISTANT_EVAL_PERSONA_DEPARTED"


def test_only_an_unfinished_run_can_be_resumed() -> None:
    """🔴 A cancelled run is one an operator stopped, usually to stop spend."""
    from app.assistant.evaluation.constants import (
        RESUMABLE_STATUSES,
        RUN_CANCELLED,
        RUN_FAILED,
        RUN_PARTIAL,
        RUN_SUCCEEDED,
    )

    assert RUN_PARTIAL in RESUMABLE_STATUSES
    assert RUN_FAILED in RESUMABLE_STATUSES
    assert RUN_CANCELLED not in RESUMABLE_STATUSES
    assert RUN_SUCCEEDED not in RESUMABLE_STATUSES