"""助手现在够得到安全答案库了 —— 两个入口一起。 这条洞的形状很具体:答案库的 71 条全部是客户安全问卷的口吻,而员工问的是 「我能用个人电脑写公司代码吗」。员工端的安全 FAQ 页为此长出了混合检索,而**助手 根本答不了这个问题** —— 它会给一件公司早就写下来的事开一张工单。 工具装在共享内核(`copilot.TOOLS`)上而不是飞书那一侧,所以网页 IT 求助和飞书 bot 同时拿到。这一批钉住的是: * 两个入口都真的看得到它(装在共享内核上,不是某一个渠道的 extra_tools); * 它**严格只读** —— 一个模型一轮里可能调它三次,而一个「读」工具去写库正是本仓 savepoint 闸存在的那个形状; * 它只返回员工可见的条目; * 两边的提示词都告诉了模型它存在,而且「没有依据就明说」这条对安全问题最严格。 """ from __future__ import annotations import json import uuid import pytest from app.domains.support.copilot import SECURITY_TOOL_NAME pytestmark = pytest.mark.asyncio async def _entry(db, *, visible: bool = True, zh: str | None = None): from app.domains.auditor.models import AuditorAnswerLibraryEntry row = AuditorAnswerLibraryEntry( id=uuid.uuid4(), category="general", question_zh=zh or "贵公司是否允许员工使用个人设备处理公司数据?", question_en="Does your company permit BYOD for company data?", answer_zh="公司允许在满足 MDM 纳管的前提下使用个人设备处理公司数据。", answer_en="BYOD is permitted when the device is MDM-enrolled.", status="active", employee_visible=visible, ) db.add(row) await db.flush() return row async def _person(db): from app.identity.person_models import IdentityPerson person = IdentityPerson( id=uuid.uuid4(), source_system="successfactors", hris_person_id=f"P{uuid.uuid4().hex[:8]}", display_name="提问的员工", work_email=f"u{uuid.uuid4().hex[:8]}@pingcap.com", employment_status="active", ) db.add(person) await db.flush() return person # ── 装在哪 ────────────────────────────────────────────────────────── def test_the_tool_lives_on_the_shared_kernel(): """🔴 两个入口一起吃到,是这条洞的修法本身。 装在飞书那一侧的 ``extra_tools`` 上,网页 IT 求助就仍然答不了安全问题 —— 而那是同一个问题的另一半。 """ from app.domains.support.copilot import TOOLS names = [tool["name"] for tool in TOOLS] assert SECURITY_TOOL_NAME in names def test_both_prompts_tell_the_model_it_exists(): """一个模型不知道存在的工具等于不存在。""" from app.assistant.prompt import feishu_system_prompt from app.domains.support.copilot import SYSTEM_PROMPT feishu = feishu_system_prompt("zh") assert SECURITY_TOOL_NAME in feishu # 网页那一份没有把工具名插进去(它的措辞是「the security-policy tool」), # 所以这里断言的是意图而不是字面。 assert "security-policy tool" in SYSTEM_PROMPT assert "on the company's behalf" in SYSTEM_PROMPT def test_the_feishu_prompt_routes_permission_questions_to_it(): """「能不能…」是权限问题,不是操作问题 —— 路由顺序里要认得出来。""" from app.assistant.prompt import feishu_system_prompt text = feishu_system_prompt("zh") assert "POLICY question" in text # 而且要说清它和知识库不是同一件事,否则模型会以为查过 KB 就够了。 assert "a how-to article is not a policy statement" in text # ── 行为 ──────────────────────────────────────────────────────────── async def test_it_answers_from_the_library(db_session): from app.domains.support.copilot import _dispatch_tool await _entry(db_session) person = await _person(db_session) await db_session.commit() raw = await _dispatch_tool( SECURITY_TOOL_NAME, {"question": "个人设备处理公司数据可以吗"}, db_session, person, ) payload = json.loads(raw) assert payload["covered"] is True assert payload["entries"], "没有召回到任何条目" assert "MDM" in payload["entries"][0]["answer"] async def test_it_never_returns_an_entry_employees_cannot_see(db_session): """🔴 这个工具是员工在用,答案库里有不对员工公开的条目。""" from app.domains.support.copilot import _dispatch_tool await _entry(db_session, visible=False, zh="贵公司的内部渗透测试频率?") person = await _person(db_session) await db_session.commit() raw = await _dispatch_tool( SECURITY_TOOL_NAME, {"question": "内部渗透测试频率"}, db_session, person, ) payload = json.loads(raw) assert payload["entries"] == [] assert payload["covered"] is False async def test_it_writes_nothing(db_session): """🔴 严格只读,而且这是一个决定而不是疏漏。 ``ask(record_gaps=True)`` 会往安全团队的缺口台账写一行 —— 但一个模型在一轮里 可能换着措辞调这个工具三次,那会把同一个缺口记三遍;而一个「读」工具去写库, 正是本仓 savepoint 闸存在的那个形状:这里一次失败的 insert 会在任何东西解释它 之前若干条语句就把整轮的事务毒死。 """ from sqlalchemy import func, select from app.domains.auditor.models import AuditorSecurityFaqGap from app.domains.support.copilot import _dispatch_tool await _entry(db_session) person = await _person(db_session) await db_session.commit() before = ( await db_session.execute( select(func.count()).select_from(AuditorSecurityFaqGap) ) ).scalar() # 这句话必须**一条都召回不到**,那才是 record_gaps=True 会写行的那条路。 # 第一版用的是「公司给员工报销家庭宽带吗」—— 它和库里那条共享「公司」「员工」 # 两个词,于是清过了词项线、走进了「召回到了」的分支,而那条分支只在模型说 # 「都答不上」时才写行(本地没配模型 → 不写)。也就是说那一版无论 record_gaps # 是真是假都不写行,断言是空转的。 probe = "熊猫会不会飞" from app.domains.auditor import security_faq_service as security precheck = await security.ask( db_session, query=probe, lang="zh", person_id=None, limit=1, compose=False, record_gaps=False, ) assert not precheck.get("items"), ( "前置条件不成立:这句探针被召回到了,走不到会写缺口的那条分支" ) await _dispatch_tool( SECURITY_TOOL_NAME, {"question": probe}, db_session, person, ) await db_session.commit() after = ( await db_session.execute( select(func.count()).select_from(AuditorSecurityFaqGap) ) ).scalar() assert after == before, "只读工具往缺口台账写了行" async def test_an_uncovered_question_says_so_rather_than_returning_nothing( db_session, ): """`covered: false` 是一个可用的答案 —— 模型要能据此说「库里没有」。""" from app.domains.support.copilot import _dispatch_tool person = await _person(db_session) await db_session.commit() raw = await _dispatch_tool( SECURITY_TOOL_NAME, {"question": "公司允许养宠物进办公室吗"}, db_session, person, ) payload = json.loads(raw) assert payload["covered"] is False assert "does not cover it" in payload["note"] async def test_an_empty_question_is_refused_without_a_query(db_session): from app.domains.support.copilot import _dispatch_tool person = await _person(db_session) await db_session.commit() payload = json.loads( await _dispatch_tool(SECURITY_TOOL_NAME, {"question": " "}, db_session, person) ) assert "error" in payload async def test_it_hands_the_model_answers_and_not_retrieval_internals(db_session): """🔴 把置信度分层、通道来源递给模型,它会把检索内部状态当政策念给员工。""" from app.domains.support.copilot import _dispatch_tool await _entry(db_session) person = await _person(db_session) await db_session.commit() payload = json.loads( await _dispatch_tool( SECURITY_TOOL_NAME, {"question": "个人设备处理公司数据可以吗"}, db_session, person, ) ) assert set(payload) == {"covered", "lead_answer", "entries", "note"} assert set(payload["entries"][0]) == {"id", "question", "answer"}