"""Vocabulary of the assistant evaluation harness.

Every enum here is a *stored* string, so renaming one costs a migration plus a
backfill. They are deliberately coarse: a taxonomy nobody can apply
consistently produces a queue nobody trusts.

The failure taxonomy is **the same five buckets the knowledge-gap ledger
already uses** (:mod:`app.assistant.gap_verification`), for one reason: a
failing evaluation case and an employee's 👎 are the same kind of object — a
question the assistant got wrong — and they must land in one worklist, not two
with different words for the same defect. Two members are added that a
human-reported gap cannot express (:data:`FAIL_JUDGE_ERROR`,
:data:`FAIL_REGRESSION`) because they describe the harness's own findings.
"""
from __future__ import annotations

# --------------------------------------------------------------------------
# Suites and cases
# --------------------------------------------------------------------------

SUITE_ACTIVE = "active"
SUITE_ARCHIVED = "archived"
SUITE_STATUSES = (SUITE_ACTIVE, SUITE_ARCHIVED)

#: A case is only executed when it is ``active``. ``draft`` is the state every
#: machine-produced case starts in — mined from a real transcript, generated as
#: a variant, or promoted from a failure — because a question nobody read is
#: not a specification, and an unreviewed mined question may still carry the
#: asker's own data. ``retired`` keeps history readable without re-running a
#: case whose answer the product deliberately changed.
CASE_DRAFT = "draft"
CASE_ACTIVE = "active"
CASE_RETIRED = "retired"
CASE_STATUSES = (CASE_DRAFT, CASE_ACTIVE, CASE_RETIRED)

#: Where a case came from. Kept because the useful reviews differ: a
#: ``mined`` case needs a privacy read, a ``generated`` one needs a plausibility
#: read, and a ``curated`` one is already somebody's considered opinion.
ORIGIN_CURATED = "curated"
ORIGIN_MINED = "mined"
ORIGIN_GENERATED = "generated"
ORIGIN_GAP = "gap"
ORIGIN_EVAL_FAILURE = "eval_failure"
ORIGINS = (
    ORIGIN_CURATED,
    ORIGIN_MINED,
    ORIGIN_GENERATED,
    ORIGIN_GAP,
    ORIGIN_EVAL_FAILURE,
)

#: What the assistant is being asked to do. This is the axis worth reporting
#: on: "the assistant is at 82%" is not actionable, "action proposal is at 55%
#: while KB lookup is at 94%" is.
CAT_KB_LOOKUP = "kb_lookup"
CAT_FLOW_DISCOVERY = "flow_discovery"
CAT_ACTION_PROPOSAL = "action_proposal"
CAT_SELF_DATA = "self_data"
CAT_SECURITY_POLICY = "security_policy"
CAT_OUT_OF_SCOPE = "out_of_scope"
CAT_AMBIGUOUS = "ambiguous"
CAT_MULTI_INTENT = "multi_intent"
CAT_SAFETY = "safety"
#: 员工点名了一个**具体对象**(云账号号码、实例 id、IP、域名、资产序列号),
#: 答案必须是那个对象的信息。单独成一类是因为它的失败方式独一无二:回答**看起来
#: 完全合理**,只是说的是另一个东西 —— 2026-09-07 生产上「380838443567 这个云账号
#: 谁负责」被答成了两条同名审批流程的负责人,每一句都对,合起来全错。
CAT_OBJECT_LOOKUP = "object_lookup"
CATEGORIES = (
    CAT_KB_LOOKUP,
    CAT_FLOW_DISCOVERY,
    CAT_ACTION_PROPOSAL,
    CAT_SELF_DATA,
    CAT_SECURITY_POLICY,
    CAT_OUT_OF_SCOPE,
    CAT_AMBIGUOUS,
    CAT_MULTI_INTENT,
    CAT_SAFETY,
    CAT_OBJECT_LOOKUP,
)

LANGS = ("zh", "en", "ja")

# --------------------------------------------------------------------------
# Runs
# --------------------------------------------------------------------------

TRIGGER_MANUAL = "manual"
TRIGGER_SCHEDULED = "scheduled"
TRIGGER_MCP = "mcp"
TRIGGERS = (TRIGGER_MANUAL, TRIGGER_SCHEDULED, TRIGGER_MCP)

RUN_RUNNING = "running"
RUN_SUCCEEDED = "succeeded"
#: Ran to the end of its budget rather than the end of its case list. A partial
#: run is a legitimate result and its numbers are real for the cases it did
#: reach — it is NOT an error, and hiding it as one is how a harness starts
#: silently measuring a shrinking subset.
RUN_PARTIAL = "partial"
RUN_FAILED = "failed"
RUN_CANCELLED = "cancelled"
RUN_STATUSES = (RUN_RUNNING, RUN_SUCCEEDED, RUN_PARTIAL, RUN_FAILED, RUN_CANCELLED)
RUN_TERMINAL_STATUSES = (RUN_SUCCEEDED, RUN_PARTIAL, RUN_FAILED, RUN_CANCELLED)
#: The only statuses ``/runs/{id}/resume`` may revive. Deliberately NOT "every
#: status except running": ``cancelled`` is a run an operator stopped on
#: purpose — usually to stop provider spend — and resuming it spends exactly
#: what they stopped; ``succeeded`` has no pending attempts, so a resume only
#: clears its notes and re-aggregates.
RESUMABLE_STATUSES = (RUN_PARTIAL, RUN_FAILED)

# --------------------------------------------------------------------------
# Per-attempt verdict
# --------------------------------------------------------------------------

VERDICT_PASS = "pass"
VERDICT_FAIL = "fail"
#: The harness could not obtain a verdict: the turn itself errored, or the
#: judge did. Counted separately from ``fail`` on purpose — an unreachable
#: provider is not evidence that the assistant answers badly, and folding the
#: two together makes an outage look like a quality regression.
VERDICT_ERROR = "error"
VERDICTS = (VERDICT_PASS, VERDICT_FAIL, VERDICT_ERROR)

# --------------------------------------------------------------------------
# Failure taxonomy — shared with the knowledge-gap ledger
# --------------------------------------------------------------------------

FAIL_KB_DOC = "kb_doc"
FAIL_RETRIEVAL = "retrieval"
FAIL_TOOL_FIELD_OR_DATA = "tool_field_or_data"
FAIL_ACTION = "action"
FAIL_OUT_OF_SCOPE = "out_of_scope"
#: The harness's own two.
FAIL_REGRESSION = "regression"
FAIL_JUDGE_ERROR = "judge_error"
#: A failure nothing could name. Reached when a blocking check fired that does
#: not say WHY on its own — a missing phrase, the wrong language — and the
#: judge, which is blind to the checks, voted pass and so contributed no type
#: either. It exists because the alternative was worse: those rows used to be
#: skipped by the breakdown entirely, so a run reporting ten failures rendered
#: 「这次没有失败的尝试。」 in the panel next to the count. This bucket is
#: deliberately NOT actionable — a proposal generated from "we do not know what
#: went wrong" is the confident wrong guess the taxonomy exists to avoid.
FAIL_UNCLASSIFIED = "unclassified"
FAILURE_TYPES = (
    FAIL_KB_DOC,
    FAIL_RETRIEVAL,
    FAIL_TOOL_FIELD_OR_DATA,
    FAIL_ACTION,
    FAIL_OUT_OF_SCOPE,
    FAIL_REGRESSION,
    FAIL_JUDGE_ERROR,
    FAIL_UNCLASSIFIED,
)

#: Failure types a knowledge-loop proposal can be generated for. The other
#: three are read by a human first: ``out_of_scope`` is a product decision,
#: ``judge_error`` is a harness defect, and ``regression`` names a change
#: somewhere else that has to be found before anything is written.
FAILURE_TYPES_ACTIONABLE = (FAIL_KB_DOC, FAIL_RETRIEVAL)

# --------------------------------------------------------------------------
# Judge dimensions
# --------------------------------------------------------------------------

#: Scored 0–5 by the judge model. Ordered as the UI renders them.
DIM_GROUNDED = "grounded"
DIM_ACCURATE = "accurate"
DIM_COMPLETE = "complete"
DIM_ACTIONABLE = "actionable"
DIM_ESCALATION = "escalation"
DIM_TONE = "tone"
JUDGE_DIMENSIONS = (
    DIM_GROUNDED,
    DIM_ACCURATE,
    DIM_COMPLETE,
    DIM_ACTIONABLE,
    DIM_ESCALATION,
    DIM_TONE,
)
DIMENSION_MAX = 5

#: A dimension at or below this sinks the attempt regardless of the judge's own
#: verdict field. Enforced server-side so "pass" cannot mean "the model said
#: pass while scoring groundedness 1".
DIMENSION_FAIL_AT_OR_BELOW = 2

#: Dimensions that are hard gates: an attempt scoring at or below
#: :data:`DIMENSION_FAIL_AT_OR_BELOW` on any of these fails even if every other
#: dimension is perfect. Groundedness and accuracy are what make an answer safe
#: to show an employee; the rest are quality.
JUDGE_HARD_DIMENSIONS = (DIM_GROUNDED, DIM_ACCURATE)

# --------------------------------------------------------------------------
# Deterministic check codes
# --------------------------------------------------------------------------
#
# These run BEFORE the judge and are not opinions: each one is a fact about the
# transcript that can be established without a model. A ``blocking`` finding
# fails the attempt on its own — the judge is still asked (its reasoning is
# what makes the failure diagnosable), but it cannot vote the attempt back up.

CHECK_EMPTY_REPLY = "empty_reply"
CHECK_TURN_FAILED = "turn_failed"
CHECK_UNAVAILABLE = "unavailable"
CHECK_UNGROUNDED_CITATION = "ungrounded_citation"
CHECK_FORBIDDEN_PHRASE = "forbidden_phrase"
CHECK_MISSING_PHRASE = "missing_phrase"
CHECK_MISSING_TOOL = "missing_tool"
CHECK_UNEXPECTED_ACTION = "unexpected_action"
CHECK_MISSING_ACTION = "missing_action"
CHECK_WRONG_LANGUAGE = "wrong_language"
CHECK_POLICY_CONFLICT = "policy_conflict"
CHECK_UNRESOLVED_PLACEHOLDER = "unresolved_placeholder"

#: Findings that fail the attempt by themselves.
BLOCKING_CHECKS = frozenset(
    {
        CHECK_EMPTY_REPLY,
        CHECK_UNGROUNDED_CITATION,
        CHECK_FORBIDDEN_PHRASE,
        CHECK_MISSING_PHRASE,
        CHECK_MISSING_TOOL,
        CHECK_UNEXPECTED_ACTION,
        CHECK_MISSING_ACTION,
        CHECK_WRONG_LANGUAGE,
        CHECK_UNRESOLVED_PLACEHOLDER,
    }
)

#: Findings that mark the attempt an ``error`` rather than a ``fail``: the
#: assistant never got to answer, so nothing about its quality was observed.
ERROR_CHECKS = frozenset({CHECK_TURN_FAILED, CHECK_UNAVAILABLE})

# --------------------------------------------------------------------------
# Budgets
# --------------------------------------------------------------------------

#: One attempt = one assistant turn + one judge call. The assistant turn has
#: its own 110s internal budget (``_COPILOT_TURN_BUDGET_SECONDS``); this bounds
#: the pair so one wedged case cannot eat a whole run's wall clock.
ATTEMPT_TIMEOUT_SECONDS = 180.0
JUDGE_TIMEOUT_SECONDS = 60.0

#: How many attempts run at once. The provider is shared with the live employee
#: channel, so this is deliberately modest: a regression run must never be the
#: reason an employee's question is slow.
DEFAULT_CONCURRENCY = 3
MAX_CONCURRENCY = 8

#: Wall-clock ceiling for one run. Reaching it yields ``partial``, never a
#: crash, and the run reports how many cases it did not reach.
DEFAULT_RUN_BUDGET_SECONDS = 3600.0
MAX_RUN_BUDGET_SECONDS = 10800.0

#: Repeats per case within one run.
#:
#: **Why the default is 3 and the UI pushes 6.** A pass rate read off ONE
#: attempt per case is not an estimate, it is a sample of size one against a
#: non-deterministic system: measured on this platform's own probes, three runs
#: were not enough to separate a real change from sampling noise. Repeats are
#: what turn "82%" into "82% ± 4, and these two cases are flaky". A case whose
#: attempts disagree is the single most valuable row in a report and is
#: invisible at ``repeats=1``.
DEFAULT_REPEATS = 3
RECOMMENDED_REPEATS = 6
MAX_REPEATS = 10

#: Cases per run. A ceiling exists so a suite that grows without anyone
#: noticing cannot turn the weekly cron into a six-hour provider bill.
MAX_CASES_PER_RUN = 400

# --------------------------------------------------------------------------
# Regression alerting
# --------------------------------------------------------------------------

#: Absolute pass-rate drop, in percentage points, versus the previous terminal
#: run of the same suite, that makes the scheduled job notify IT.
REGRESSION_ALERT_DROP_PP = 10.0

#: Never alert on a comparison drawn from fewer attempts than this — a run that
#: only reached nine attempts before its budget expired can "drop 20 points"
#: without anything having changed.
REGRESSION_MIN_ATTEMPTS = 20

__all__ = [name for name in dir() if not name.startswith("_")]