"""Failing provisioning tasks. A provisioning task is the platform's most consequential automation: it is what actually creates a new hire's accounts and takes a leaver's away. Two states count as failing, and including the second is the point of this collector. ``status='failed'`` is the obvious one. The other is a task sitting in ``pending`` **after** a failed attempt — the dispatcher's transient backoff. Those are invisible today: a Zoom 429 that succeeds on the third attempt leaves no trace anywhere, so the platform's own failure counts are a survivorship sample. The ledger records the episode and then resolves it on its own when the retry lands, which is exactly the shape that answers "how often does this vendor actually refuse us" without asking a human to do anything. The pending-with-backoff test is deliberately narrow: ``attempt_count > 0`` **and** an error message **and** ``not_before`` still in the future. A freshly created task also sits in ``pending`` with no attempts, and a task whose backoff has elapsed is simply waiting for the next dispatcher tick — neither is a failure. Recovery is by absence, and safely so: the scan is unbounded and returns every task currently in either failing state, so a task that has left them has genuinely stopped failing (it completed, was cancelled, or its retry succeeded). """ from __future__ import annotations import uuid from datetime import UTC, datetime from sqlalchemy import or_, select from sqlalchemy.ext.asyncio import AsyncSession from app.core.person_scope import ( EMPLOYMENT_LEAVER_PENDING, EMPLOYMENT_LEFT, EMPLOYMENT_PRE_HIRE, ) from app.domains.automation_exception.collectors.base import ( Candidate, CollectorResult, ) from app.domains.automation_exception.timeutil import newest from app.domains.provisioning.models import ( STATUS_FAILED, STATUS_PENDING, ProvisioningTask, SoftwareConnector, ) from app.identity.person_models import IdentityPerson SOURCE_REF_TYPE = "provisioning_task" #: Employment states that make a failed revoke a security fact rather #: than an inconvenience. _LEAVER_STATES = frozenset({EMPLOYMENT_LEAVER_PENDING, EMPLOYMENT_LEFT}) async def collect(db: AsyncSession, *, now: datetime | None = None) -> CollectorResult: now = now or datetime.now(UTC) rows = ( await db.execute( select(ProvisioningTask).where( or_( ProvisioningTask.status == STATUS_FAILED, # Transient backoff — the last attempt failed and the # next one is still in the future. (ProvisioningTask.status == STATUS_PENDING) & (ProvisioningTask.attempt_count > 0) & (ProvisioningTask.error_message.is_not(None)) & (ProvisioningTask.not_before.is_not(None)) & (ProvisioningTask.not_before > now), ), ), ) ).scalars().all() if not rows: return CollectorResult( candidates=[], absence_means_recovery=True, ) # Employment state of every subject, in one query — it decides # severity and must not cost a round trip per task. person_ids = {r.user_id for r in rows if r.user_id} employment: dict[uuid.UUID, str] = {} if person_ids: emp_rows = await db.execute( select(IdentityPerson.id, IdentityPerson.employment_status).where( IdentityPerson.id.in_(sorted(person_ids, key=str)), ), ) employment = {row[0]: row[1] for row in emp_rows.all()} # software_id -> integration_id, via the active connector binding. # This is what lets a failed seat grant join the same cluster as the # connection test for the same vendor. software_ids = {r.software_id for r in rows if r.software_id} integration_by_software: dict[uuid.UUID, uuid.UUID] = {} if software_ids: conn_rows = await db.execute( select( SoftwareConnector.software_id, SoftwareConnector.integration_id, ).where( SoftwareConnector.software_id.in_(sorted(software_ids, key=str)), SoftwareConnector.is_active.is_(True), ), ) for software_id, integration_id in conn_rows.all(): integration_by_software.setdefault(software_id, integration_id) candidates: list[Candidate] = [] for row in rows: emp = employment.get(row.user_id) if row.user_id else None integration_id = ( integration_by_software.get(row.software_id) if row.software_id else None ) # ``updated_at`` is when the attempt failed; ``created_at`` is # when the work was queued. The failure's time is the one that # belongs on an occurrence. occurred_at = newest(row.updated_at, row.created_at) or now candidates.append( Candidate( source_ref_type=SOURCE_REF_TYPE, source_ref_id=str(row.id), occurred_at=occurred_at, error_text=row.error_message, # Scope by the software package when we know it: the same # vendor sentence about two different packages is two # different fixes. scope_key=( f"software:{row.software_id}" if row.software_id else f"target:{row.target_type}" ), detail={ "action": row.action, "target_type": row.target_type, "execution_mode": row.execution_mode, "status": row.status, "attempt_count": row.attempt_count, "retry_pending": row.status == STATUS_PENDING, "not_before": ( row.not_before.isoformat() if row.not_before else None ), }, person_id=row.user_id, integration_id=integration_id, software_id=row.software_id, # The task row carries everything needed to run it again, # and the service already has a guarded retry entry point. replayable=True, severity_facts={ "action": row.action, "subject_is_leaver": emp in _LEAVER_STATES, "subject_is_pending_hire": emp == EMPLOYMENT_PRE_HIRE, }, ), ) return CollectorResult( candidates=candidates, # Safe: the scan above is the complete current failing set, with # no time bound, so an absent task really has stopped failing. absence_means_recovery=True, )