"""Integrations whose last connection test failed. This collector closes the gap the audit named: ``last_test_status`` is written **only** when an admin clicks "test connection" by hand. Nothing in the scheduler ever touches it. So an expired credential is never noticed at the integration; it is noticed as a rising tide of failures in five downstream domains, each of which reports the symptom. The other half of the fix is ``integrations.health_check``, the daily cron added alongside this collector, which actually performs the tests. Without it this collector would only ever report what an admin had already discovered — which is the shape of a metric that looks fine because nobody is measuring. ``source_ref_id`` is the **slug**, not the row id. An integration is a long-lived, human-named thing; the slug is what an admin recognises, what the credential remedy names, and what survives a row being recreated during a re-configuration. Disabled integrations are still reported, at low severity. A disabled integration with broken credentials is not an outage — but it is also not nothing, because "we turned it off and forgot" is how a half-migrated vendor ends up holding live accounts. """ from __future__ import annotations from datetime import UTC, datetime from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.domains.automation_exception.collectors.base import ( Candidate, CollectorResult, ) from app.domains.automation_exception.timeutil import newest from app.integrations.models import Integration SOURCE_REF_TYPE = "integration" #: What ``IntegrationService.test_connection`` writes on failure. _TEST_FAILED = "failed" _TEST_OK = "ok" async def collect(db: AsyncSession, *, now: datetime | None = None) -> CollectorResult: now = now or datetime.now(UTC) rows = ( await db.execute(select(Integration)) ).scalars().all() candidates: list[Candidate] = [] recovered_refs: set[str] = set() for row in rows: if row.last_test_status == _TEST_OK: # Explicit recovery evidence: it was tested and it passed. recovered_refs.add(row.slug) continue if row.last_test_status != _TEST_FAILED: # Never tested. Not a failure — and deliberately not treated # as one: reporting "untested" as an exception would fill the # workbench on day one with rows whose remedy is "click a # button", before the health-check cron has had a night to # run. The ledger's own hygiene reporting is where an # untested integration belongs. continue candidates.append( Candidate( source_ref_type=SOURCE_REF_TYPE, source_ref_id=row.slug, occurred_at=newest(row.last_tested_at, row.updated_at) or now, error_text=row.last_test_message, # The integration IS the scope. Its own slug would be # redundant here, and an empty scope keeps the cluster # key readable: one integration, one failing credential, # one row. scope_key="", detail={ "slug": row.slug, "kind": row.kind, "provider": row.provider, "name": row.name, "enabled": bool(row.enabled), "is_primary": bool(row.is_primary), "last_tested_at": ( row.last_tested_at.isoformat() if row.last_tested_at else None ), }, person_id=None, integration_id=row.id, # A connection test is not a per-row action to replay; # PR3's credential remedy re-tests the integration and # then replays the *downstream* occurrences, which is a # different and much more useful thing. replayable=False, severity_facts={"integration_enabled": bool(row.enabled)}, ), ) return CollectorResult( candidates=candidates, recovered_refs=recovered_refs, # Every integration row was examined, so absence means the row # was deleted. Recovery is nonetheless driven by the explicit # ``ok`` evidence above, which is stronger. absence_means_recovery=True, )