"""Deterministic root-cause rules, and how bad a thing is. Two separate jobs, deliberately kept apart: * :func:`classify_cause` reads the **error text**. It is a pattern matcher and nothing more, and it answers ``None`` whenever it is not sure. PR2's triage picks up exactly the rows it declined. * :func:`compute_severity` reads **structured facts** — which source, which action, whether a hire or a leaver is waiting, which SLO tier the cron sits in. It never looks at the error text, because how bad a failure is has nothing to do with how the vendor worded it. Two design rules, both learned the hard way in this repository: **Word boundaries on every numeric code.** ``"429" in text`` matches ``"group 4290 not found"`` and ``"15034 rows"``. The provisioning transient classifier shipped with a bare substring match and had to be fixed post-review for precisely that; a misclassification there merely delayed a loud failure, whereas here it would file a not-found under "rate limited" and hand the admin the wrong remedy. **Silence beats a guess.** ``classify_cause`` returning ``None`` costs one row sitting in ``unknown`` until triage. A wrong confident verdict costs an admin doing the wrong thing, and — once PR3 lands — an automatic replay against a vendor that was never going to accept it. Every pattern below is one an admin would recognise on sight; anything arguable is left out on purpose. """ from __future__ import annotations import re from typing import Final from app.domains.automation_exception.taxonomy import ( CAUSE_CONFIG_DRIFT, CAUSE_CREDENTIAL, CAUSE_DATA_QUALITY, CAUSE_PLATFORM_DEFECT, CAUSE_TARGET_MISSING, CAUSE_THROTTLED, CAUSE_VENDOR_POLICY, SEVERITY_CRITICAL, SEVERITY_HIGH, SEVERITY_LOW, SEVERITY_NORMAL, SOURCE_IDENTITY_SYNC, SOURCE_INTEGRATION, SOURCE_NOTIFICATION, SOURCE_PROVISIONING, SOURCE_SCHEDULED_JOB, ) from app.notifications.categories import BUCKET_ACTION # -------------------------------------------------------------------- # Root cause from error text # -------------------------------------------------------------------- #: Our own stack traces and driver errors. First, and unconditional: #: a Python traceback in a vendor-error field is never the vendor's #: fault, whatever else the text happens to contain. _PLATFORM_DEFECT_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\btraceback \(most recent call last\)", r"\b(?:attribute|type|key|index|value|name)error\b", r"\b(?:programming|integrity|operational|invalid request)error\b", r"\bmissinggreenlet\b", r"\bunboundlocalerror\b", r"\bjsondecodeerror\b", r"\bnot all arguments converted\b", r"\bobject has no attribute\b", r"\bunhashable type\b", ) ) #: The credential we hold is refused. Checked before anything else #: vendor-shaped, because a 401 dressed up as "permission denied" still #: needs a new credential and not a config change. _CREDENTIAL_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\b401\b", r"\bunauthori[sz]ed\b", r"\binvalid_grant\b", r"\binvalid_client\b", r"\binvalid_token\b", r"\bbad credentials\b", # GitHub's wording r"\btoken (?:has )?expired\b", r"\bexpired (?:access )?token\b", r"\bauthentication failed\b", r"\binvalid signature\b", r"\bsignature ?does ?not ?match\b", r"\binvalid (?:api )?key\b", r"\bcredential(?:s)? (?:is |are )?(?:invalid|expired|revoked)\b", r"\bapp (?:is )?not installed\b", r"\brefresh token .{0,20}(?:invalid|expired|revoked)\b", ) ) #: Rate limited or briefly unavailable. Mirrors the vocabulary the #: provisioning dispatcher already treats as transient, so the two #: layers agree about what "it will probably work later" means. _THROTTLED_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\b429\b", r"\b50[234]\b", r"\btoo many (?:concurrent )?requests\b", r"\brate[ _-]?limit", r"\bthrottl", r"\btimed out\b", r"\btimeout\b", r"\bbad gateway\b", r"\bservice unavailable\b", r"\btemporarily unavailable\b", r"\bconnection (?:reset|refused|aborted)\b", r"\bserver is busy\b", ) ) #: The vendor's own commercial or plan rules said no. Money or a vendor #: console setting, never our code and never a retry. _VENDOR_POLICY_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\bquota (?:exceeded|exhausted)\b", r"\b(?:seat|user|licen[cs]e) limit\b", r"\bexceed(?:s|ed)? .{0,24}limit\b", r"\bno (?:available )?licen[cs]e", r"\blicen[cs]e (?:is )?(?:required|missing|not assigned)\b", r"\bnot (?:available|included) in (?:your |the )?plan\b", r"\bplan (?:does not|doesn't) (?:support|include)\b", r"\bupgrade your (?:plan|subscription)\b", r"\bsubscription (?:expired|inactive|required)\b", r"\bplus_teams_only\b", # Slack SCIM on the wrong plan r"\bmax(?:imum)? number of users\b", r"\binsufficient balance\b", r"\bfree quota .{0,20}(?:used up|exhausted)\b", ) ) #: The subject is not there. Kept distinct from config drift because #: the remedy is usually "confirm it was already gone", i.e. a no-op — #: and for a revoke, a missing target means the goal is already met. _TARGET_MISSING_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\b404\b", # The id usually sits between the noun and the verdict # ("group 4290 not found", "user ou_abc not found"), so allow a # bounded gap. Bounded, not greedy: an unbounded ``.*`` would # let a multi-line traceback that merely contains the word # "user" pair with a "not found" hundreds of characters later. r"\b(?:user|member|account|group|resource|record|entity|mailbox)\b" r".{0,48}?\bnot found\b", r"\bdoes ?not exist\b", r"\bno such (?:user|member|account|group|entity|resource)\b", r"\bnot a member\b", r"\balready (?:removed|deleted|deactivated|suspended)\b", r"\bresource ?not ?found\b", ) ) #: Our configuration disagrees with the vendor's reality. _CONFIG_DRIFT_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\bunknown (?:field|attribute|parameter|scope)\b", r"\bunsupported (?:field|attribute|operation|parameter)\b", r"\binvalid (?:scope|endpoint|region|project|workspace|tenant)\b", r"\bscope .{0,24}not (?:granted|authori[sz]ed)\b", r"\bmissing (?:required )?scope\b", r"\bnot configured\b", r"\bno (?:connector|binding|mapping) configured\b", r"\bendpoint .{0,20}not (?:found|supported)\b", r"\bapi (?:has been )?(?:deprecated|removed)\b", ) ) #: The payload we sent was not acceptable. _DATA_QUALITY_PATTERNS: Final[tuple[re.Pattern[str], ...]] = tuple( re.compile(p, re.IGNORECASE) for p in ( r"\balready exists\b", r"\bduplicate\b", r"\b(?:is )?required (?:field|attribute|parameter)\b", r"\bmissing required\b", r"\binvalid (?:email|phone|mobile|format|value|argument|character)\b", r"\bvalidation (?:failed|error)\b", r"\bmalformed\b", r"\btoo (?:long|short)\b", r"\bmust (?:be|match|contain)\b", r"\bnot a valid\b", r"\bconflict(?:s)? with (?:an )?existing\b", ) ) #: Evaluation order. First match wins, so this tuple is the rule set's #: precedence and the only place it is expressed. #: #: * platform defects first — a traceback settles it outright. #: * credential before everything vendor-shaped: a refused credential #: often *also* mentions permissions or a missing resource, and the #: only fix that works is a new credential. #: * throttling next: ``429`` / ``503`` are unambiguous, and treating a #: transient as anything else sends an admin chasing a fix for #: something already recovering. #: * vendor policy before target-missing and config-drift: "no licence #: available" tends to arrive alongside a not-found for the account #: that consequently was never created. #: * data quality last of the vendor causes: its vocabulary #: ("duplicate", "already exists") overlaps target-missing's, and #: when both appear the more specific cause is the earlier one. _CAUSE_ORDER: Final[tuple[tuple[str, tuple[re.Pattern[str], ...]], ...]] = ( (CAUSE_PLATFORM_DEFECT, _PLATFORM_DEFECT_PATTERNS), (CAUSE_CREDENTIAL, _CREDENTIAL_PATTERNS), (CAUSE_THROTTLED, _THROTTLED_PATTERNS), (CAUSE_VENDOR_POLICY, _VENDOR_POLICY_PATTERNS), (CAUSE_TARGET_MISSING, _TARGET_MISSING_PATTERNS), (CAUSE_CONFIG_DRIFT, _CONFIG_DRIFT_PATTERNS), (CAUSE_DATA_QUALITY, _DATA_QUALITY_PATTERNS), ) def classify_cause(error_text: str | None) -> str | None: """Root cause for ``error_text``, or ``None`` when unsure. ``None`` is a first-class answer and the common one for novel vendor wording. Callers store :data:`~app.domains.automation_exception.taxonomy.CAUSE_UNKNOWN` and leave the row for triage. """ if not error_text or not error_text.strip(): return None text = error_text[:2000] for cause, patterns in _CAUSE_ORDER: for pattern in patterns: if pattern.search(text): return cause return None # -------------------------------------------------------------------- # Severity from structured facts # -------------------------------------------------------------------- #: Provisioning actions whose failure leaves access in place. A revoke #: that did not happen is a security fact, not an inconvenience. _REVOKE_ACTIONS: Final[frozenset[str]] = frozenset({"revoke", "remove", "disable"}) #: Notification buckets whose delivery someone is waiting on. Info and #: social traffic failing is a quality issue, not an incident. #: #: Imported, never retyped. An earlier draft spelled these ``actionable`` #: and ``critical`` — plausible words that appear nowhere in the #: notifications domain, whose real vocabulary is ``action`` / ``info`` / #: ``social``. The set would have matched nothing and quietly rated every #: undelivered actionable notification as ``normal``. _ACTIONABLE_NOTIFICATION_BUCKETS: Final[frozenset[str]] = frozenset({BUCKET_ACTION}) def compute_severity( *, source: str, root_cause: str | None = None, action: str | None = None, subject_is_pending_hire: bool = False, subject_is_leaver: bool = False, notification_bucket: str | None = None, slo_tier: str | None = None, integration_enabled: bool = True, affected_person_count: int = 0, ) -> str: """How loud this exception should be. Reads facts, never text. Each branch below names the person or the invariant at risk, which is the only defensible basis for calling something critical. """ # A refused credential on a live integration takes everything # downstream with it, whether or not any individual row has failed # yet. This is the one cause that is critical on its own. if ( source == SOURCE_INTEGRATION and root_cause == CAUSE_CREDENTIAL and integration_enabled ): return SEVERITY_CRITICAL if source == SOURCE_INTEGRATION: # A broken connection test on a disabled integration is a # ledger-hygiene fact, not an outage. return SEVERITY_HIGH if integration_enabled else SEVERITY_LOW if source == SOURCE_PROVISIONING: # Access that should be gone but is not. if action and action.lower() in _REVOKE_ACTIONS and subject_is_leaver: return SEVERITY_CRITICAL # A hire whose account has not been created yet. if subject_is_pending_hire: return SEVERITY_CRITICAL return SEVERITY_HIGH if source == SOURCE_IDENTITY_SYNC: # An account that should be disabled downstream and is not. if subject_is_leaver: return SEVERITY_CRITICAL return SEVERITY_HIGH if source == SOURCE_NOTIFICATION: if notification_bucket in _ACTIONABLE_NOTIFICATION_BUCKETS: return SEVERITY_HIGH return SEVERITY_NORMAL if source == SOURCE_SCHEDULED_JOB: if slo_tier == "P0": return SEVERITY_HIGH if slo_tier == "P1": return SEVERITY_NORMAL return SEVERITY_LOW if affected_person_count == 0 else SEVERITY_NORMAL return SEVERITY_NORMAL