"""Admin alerting when a best-effort Feishu approval cancel fails. Every requester-facing withdraw flow (DNS record requests, SSO grants, VPN applications, announcement requests, AI tool applications, domain registrations, cloud scope requests, asset employee requests) follows the same shape: flip the local row to ``cancelled``, then best-effort ``POST /approval/v4/instances/cancel`` on the parallel Feishu instance. A Feishu-side failure (e.g. code 60009 — the approval template has 「允许撤销审批中的申请」 disabled) must not block the local cancel, but until 2026-06-10 it was ALSO invisible: the requester saw 撤回成功, the Feishu card stayed 审批中, and approvers kept receiving todos. This module is the shared "make the desync visible" primitive: * the calling service records the outcome on its ``last_feishu_cancel_synced`` attribute so routers / MCP tools can surface ``feishu_cancel_synced`` to the requester (frontend toasts a warning when it comes back ``false``); * :func:`notify_feishu_cancel_desync` pushes an in-app notification (outbox row only — no Feishu/email channel push, because the Feishu API is exactly what just failed) to every active global admin, carrying the numeric Feishu error code + a remediation hint; * dedup per ``resource_type`` over a 24h window so a broken template setting (which fails EVERY withdraw of that flow) lands one admin notification per flow per day, not one per click. Best-effort all the way down: :func:`notify_feishu_cancel_desync` never raises into the caller's withdraw path. """ from __future__ import annotations import logging import uuid from datetime import UTC, datetime, timedelta from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.approval.feishu_client import parse_feishu_error_code from app.core.exceptions import AppException, format_exception_for_log from app.core.safe_logging import sanitize_log_text from app.core.txn import best_effort from app.notifications.models import NotificationOutbox from app.notifications.service import ( _SEE_TEMPLATE_SENTINEL, NotificationEnvelope, NotificationService, ) logger = logging.getLogger(__name__) # template_key stamped on the outbox rows — both the dedup query below # and tests key on it. DESYNC_TEMPLATE_KEY = "approval_feishu_cancel_desync" _DEDUP_WINDOW = timedelta(hours=24) # Feishu code 60009 = the approval template doesn't allow withdrawing # an in-review instance. It's the most common cause by far (prod, # 2026-06-10, DNS template) and has a precise admin-side fix, so the # notification calls it out explicitly. _CODE_TEMPLATE_WITHDRAW_DISABLED = 60009 # Locale-fragment triples for the two parts of this alert that can't be one # shared sentence: the error code may be unavailable, and 60009 has a precise # console-side fix while everything else gets the generic "go check the # template settings". The call site picks a pair and every language reads its # own — the ``{error_code_label_*}`` / ``{remediation_*}`` slots in the # ``approval_feishu_cancel_desync`` template. _UNKNOWN_CODE_LABEL: dict[str, str] = {"zh": "未知", "en": "unknown", "ja": "不明"} _REMEDIATION_WITHDRAW_DISABLED: dict[str, str] = { "zh": ( "处理建议:错误码 60009 表示该审批模板未开启「允许撤销审批中的申请」。" "请到飞书审批后台为该模板打开此设置,并在飞书侧手动撤销/拒绝这张遗留" "审批单。" ), "en": ( "Suggested fix: error code 60009 means this approval template does " "not allow withdrawing a request that is still in review. Turn that " "setting on for the template in the Feishu approval console, then " "withdraw or reject the leftover request manually on the Feishu side." ), "ja": ( "対応方法:エラーコード 60009 は、この承認テンプレートで「審査中の" "申請の取り消し」が許可されていないことを示します。Feishu の承認管理" "画面で当該テンプレートの設定を有効にしたうえで、残っている承認申請を " "Feishu 側で手動で取り消すか却下してください。" ), } _REMEDIATION_GENERIC: dict[str, str] = { "zh": ( "处理建议:请在飞书审批后台检查该模板的撤回设置与应用权限,并手动处理" "这张遗留审批单。" ), "en": ( "Suggested fix: check the template's withdrawal setting and the app's " "permissions in the Feishu approval console, then handle the leftover " "request manually." ), "ja": ( "対応方法:Feishu の承認管理画面で、当該テンプレートの取り消し設定と" "アプリの権限をご確認のうえ、残っている承認申請を手動で処理して" "ください。" ), } def _uuid_or_none(value: str) -> uuid.UUID | None: try: return uuid.UUID(str(value)) except (ValueError, AttributeError, TypeError): return None async def notify_feishu_cancel_desync( db: AsyncSession, *, flow_label: str, resource_type: str, resource_id: str, instance_code: str, # ``BaseException`` (not ``Exception``): callers increasingly pass # ``BestEffortOutcome.error``, which is typed that way. This function only # ``isinstance``-checks for AppException and formats — both are safe for # the wider type. error: BaseException, trace_id: str = "", ) -> bool: """In-app notify global admins that Helix and Feishu diverged on a cancel. Returns True when a notification was dispatched, False when deduped or when the notify itself failed (logged, never raised). ``flow_label`` is the human flow name rendered in the notification title (admin notifications are backend-rendered Chinese, matching the failure-notifier convention). ``resource_type`` doubles as the dedup bucket. """ async with best_effort( db, label="cancel_desync best-effort step", log=logger, ) as be_86_step: feishu_code = ( parse_feishu_error_code(error) if isinstance(error, AppException) else None ) cutoff = datetime.now(UTC) - _DEDUP_WINDOW already_notified = ( await db.execute( select(NotificationOutbox.id) .where( NotificationOutbox.template_key == DESYNC_TEMPLATE_KEY, NotificationOutbox.related_resource_type == resource_type, NotificationOutbox.created_at >= cutoff, ) .limit(1) ) ).scalar_one_or_none() if already_notified is not None: logger.info( "feishu cancel-desync on %s %s (instance=%s, code=%s): " "deduped — admins already notified for this flow within 24h", resource_type, resource_id, instance_code, feishu_code, ) return False # Lazy import: failure_notifier pulls in the scheduled-jobs # registry; importing it at module scope from the approval # package risks an import cycle with handlers that import # approval clients. from app.scheduled_jobs.failure_notifier import list_admin_recipients admins = await list_admin_recipients(db) if not admins: logger.warning( "feishu cancel-desync on %s %s: no active global_admin " "recipients — desync stays log-only", resource_type, resource_id, ) return False safe_error = sanitize_log_text( format_exception_for_log(error, max_len=512), max_len=512, ) title = f"飞书撤回同步失败:{flow_label}" code_labels = ( {lang: str(feishu_code) for lang in ("zh", "en", "ja")} if feishu_code is not None else _UNKNOWN_CODE_LABEL ) remediation = ( _REMEDIATION_WITHDRAW_DISABLED if feishu_code == _CODE_TEMPLATE_WITHDRAW_DISABLED else _REMEDIATION_GENERIC ) # The bell is this notification's ONLY surface (``channels=[]``), and # ``render_inbox_body`` returns a stored non-sentinel body verbatim — # so the whole body has to come from the template, or an EN/JA admin # reads Chinese. Everything the old hardcoded body carried now travels # on the context instead. svc = NotificationService(db) for admin in admins: # channels=[] → outbox row only (NotificationCenter bell). # No Feishu/email push: the Feishu API just failed, and a # same-request push attempt would add latency + a second # failure to the requester's withdraw call. await svc.dispatch( NotificationEnvelope( recipient=admin, title=title, body_text=_SEE_TEMPLATE_SENTINEL, channels=[], template_key=DESYNC_TEMPLATE_KEY, url="/admin/approvals", context={ "flow_label": flow_label, "resource_type": resource_type, "resource_id": str(resource_id), "instance_code": instance_code, "feishu_error_code": feishu_code, "error_detail": safe_error, "error_code_label_zh": code_labels["zh"], "error_code_label_en": code_labels["en"], "error_code_label_ja": code_labels["ja"], "remediation_zh": remediation["zh"], "remediation_en": remediation["en"], "remediation_ja": remediation["ja"], }, related_resource_type=resource_type, related_resource_id=_uuid_or_none(resource_id), ), trace_id=trace_id, ) logger.info( "feishu cancel-desync on %s %s (instance=%s, code=%s): " "notified %d global_admin(s)", resource_type, resource_id, instance_code, feishu_code, len(admins), ) return True if not be_86_step.ok: logger.error( "feishu cancel-desync notify failed for %s %s " "(best-effort, error_type=%s)", resource_type, resource_id, be_86_step.error_type, ) return False