"""Dead notification-outbox rows. ``dead`` means the retry budget ran out: somebody was told something and did not get it. The outbox already exposes these on its own admin page, complete with error clustering — what it cannot do is put a Feishu credential failure next to the identity-sync failure caused by the same expired credential, which is the whole reason they come here. Two subtleties this collector has to respect. **Acknowledged dead rows are already closed.** The outbox marks a consciously-closed dead row by prefixing its ``last_error`` with ``ACK_NOTE_PREFIX``. Re-importing those would resurrect decisions an admin already made, so they are excluded — and, because the prefix also destroys the original error text, they could not be clustered meaningfully anyway. **Scope is the channel, not the template.** A dead row's fixable cause is almost always per-channel: the Feishu app's credential, the SMTP relay, a bot removed from a chat. Clustering by template would split one credential failure across forty templates and hide it. Recovery is by absence over an unbounded scan: a row that is no longer ``dead``-and-unacknowledged has either been resent successfully or been consciously closed, and both are recoveries as far as the ledger is concerned. """ from __future__ import annotations from datetime import UTC, datetime from sqlalchemy import or_, select from sqlalchemy.ext.asyncio import AsyncSession from app.domains.automation_exception.collectors.base import ( Candidate, CollectorResult, ) from app.domains.automation_exception.timeutil import newest from app.integrations.models import Integration from app.notifications.categories import bucket_for from app.notifications.models import ( ACK_NOTE_PREFIX, STATUS_DEAD, NotificationOutbox, ) SOURCE_REF_TYPE = "notification_outbox" #: Channels whose failures point at one configured integration, so a #: credential fix can name its blast radius. Email leaves the #: integration NULL on purpose: the SMTP/Gmail path is not modelled as #: an integration row, and inventing one would make the credential #: remedy claim a scope it cannot act on. _FEISHU_CHANNELS = frozenset({"feishu", "lark", "im"}) _FEISHU_INTEGRATION_KIND = "chat_approval" async def collect(db: AsyncSession, *, now: datetime | None = None) -> CollectorResult: now = now or datetime.now(UTC) rows = ( await db.execute( select(NotificationOutbox).where( NotificationOutbox.status == STATUS_DEAD, # Not consciously closed by an admin. or_( NotificationOutbox.last_error.is_(None), ~NotificationOutbox.last_error.startswith(ACK_NOTE_PREFIX), ), ), ) ).scalars().all() if not rows: return CollectorResult(candidates=[], absence_means_recovery=True) feishu_integration = ( await db.execute( select(Integration).where( Integration.kind == _FEISHU_INTEGRATION_KIND, Integration.is_primary.is_(True), ).limit(1), ) ).scalar_one_or_none() candidates: list[Candidate] = [] for row in rows: failed = [str(c) for c in (row.failed_channels or [])] if not failed: # ``dead`` with no per-channel detail: the row exhausted its # attempts before any channel reported. Cluster it under the # requested channels so it is still attributable. failed = [str(c) for c in (row.channels or [])] or ["unknown"] # One occurrence per row, scoped to its primary failing channel. # A row that failed on two channels is one undelivered message, # not two, and the first failing channel is the one to fix. channel = failed[0] integration_id = ( feishu_integration.id if channel in _FEISHU_CHANNELS and feishu_integration is not None else None ) candidates.append( Candidate( source_ref_type=SOURCE_REF_TYPE, source_ref_id=str(row.id), occurred_at=newest(row.updated_at, row.created_at) or now, error_text=row.last_error, scope_key=f"channel:{channel}", detail={ "channels": [str(c) for c in (row.channels or [])], "failed_channels": failed, "template_key": row.template_key, "attempts": row.attempts, "max_attempts": row.max_attempts, "related_resource_type": row.related_resource_type, }, person_id=row.recipient_person_id, integration_id=integration_id, # The outbox already exposes a guarded admin resend that # rebuilds the envelope from the row. replayable=True, # ``bucket_for`` is the notifications domain's own # template-key → bucket resolver, the same one the inbox # groups by. An earlier draft of this collector read a # ``bucket`` key out of ``outbox.context``; no such key # is ever written, so it would have quietly reported # ``None`` for every row and made every dead # notification look like low-stakes social traffic. severity_facts={"notification_bucket": bucket_for(row.template_key)}, ), ) return CollectorResult( candidates=candidates, # Unbounded scan of the complete dead-and-unacknowledged set. absence_means_recovery=True, )